| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.0, |
| "eval_steps": 100, |
| "global_step": 5510, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0, |
| "grad_norm": 41.204517024742586, |
| "learning_rate": 2.7223230490018146e-10, |
| "logits/chosen": -1.0541447401046753, |
| "logits/rejected": -0.7520447373390198, |
| "logps/chosen": -60.40666961669922, |
| "logps/rejected": -106.31614685058594, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.0, |
| "grad_norm": 46.36022550961656, |
| "learning_rate": 2.722323049001815e-09, |
| "logits/chosen": -1.6423306465148926, |
| "logits/rejected": -1.0551129579544067, |
| "logps/chosen": -118.1044921875, |
| "logps/rejected": -147.71112060546875, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.4444444477558136, |
| "rewards/chosen": 0.00048234747373498976, |
| "rewards/margins": -0.009559460915625095, |
| "rewards/rejected": 0.01004180870950222, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.01, |
| "grad_norm": 44.73280646725986, |
| "learning_rate": 5.44464609800363e-09, |
| "logits/chosen": -1.3452703952789307, |
| "logits/rejected": -1.4189544916152954, |
| "logps/chosen": -100.15978240966797, |
| "logps/rejected": -120.36905670166016, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.005216827150434256, |
| "rewards/margins": 0.023615092039108276, |
| "rewards/rejected": -0.02883191965520382, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.01, |
| "grad_norm": 47.03882983712326, |
| "learning_rate": 8.166969147005445e-09, |
| "logits/chosen": -1.8659166097640991, |
| "logits/rejected": -1.3837544918060303, |
| "logps/chosen": -97.74725341796875, |
| "logps/rejected": -225.3568878173828, |
| "loss": 0.6933, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.01882881112396717, |
| "rewards/margins": 0.012613847851753235, |
| "rewards/rejected": 0.0062149628065526485, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.01, |
| "grad_norm": 51.31627563719448, |
| "learning_rate": 1.088929219600726e-08, |
| "logits/chosen": -1.726585030555725, |
| "logits/rejected": -1.5602327585220337, |
| "logps/chosen": -117.8109359741211, |
| "logps/rejected": -127.14540100097656, |
| "loss": 0.6942, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.006876278668642044, |
| "rewards/margins": 0.018800906836986542, |
| "rewards/rejected": -0.025677183642983437, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.02, |
| "grad_norm": 41.50488252094809, |
| "learning_rate": 1.3611615245009074e-08, |
| "logits/chosen": -1.8296377658843994, |
| "logits/rejected": -1.4772355556488037, |
| "logps/chosen": -97.29546356201172, |
| "logps/rejected": -119.9217758178711, |
| "loss": 0.6936, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.01763664372265339, |
| "rewards/margins": 0.02796524204313755, |
| "rewards/rejected": -0.010328599251806736, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.02, |
| "grad_norm": 43.88658635517158, |
| "learning_rate": 1.633393829401089e-08, |
| "logits/chosen": -1.7242791652679443, |
| "logits/rejected": -1.7074722051620483, |
| "logps/chosen": -93.03128814697266, |
| "logps/rejected": -110.11827087402344, |
| "loss": 0.6967, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.006739559583365917, |
| "rewards/margins": -0.004187393002212048, |
| "rewards/rejected": 0.010926952585577965, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.03, |
| "grad_norm": 45.67398430745054, |
| "learning_rate": 1.90562613430127e-08, |
| "logits/chosen": -2.0839061737060547, |
| "logits/rejected": -1.6974592208862305, |
| "logps/chosen": -89.07298278808594, |
| "logps/rejected": -121.43327331542969, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.018308373168110847, |
| "rewards/margins": 0.026240844279527664, |
| "rewards/rejected": -0.007932471111416817, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.03, |
| "grad_norm": 48.69573234717181, |
| "learning_rate": 2.177858439201452e-08, |
| "logits/chosen": -1.2379311323165894, |
| "logits/rejected": -1.0085999965667725, |
| "logps/chosen": -92.49485778808594, |
| "logps/rejected": -110.10205078125, |
| "loss": 0.6901, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.012255726382136345, |
| "rewards/margins": 0.024808084592223167, |
| "rewards/rejected": -0.012552358210086823, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.03, |
| "grad_norm": 41.975191835891444, |
| "learning_rate": 2.4500907441016332e-08, |
| "logits/chosen": -1.6204273700714111, |
| "logits/rejected": -1.3888098001480103, |
| "logps/chosen": -96.46438598632812, |
| "logps/rejected": -131.0227508544922, |
| "loss": 0.694, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.0034812160301953554, |
| "rewards/margins": -0.005814972333610058, |
| "rewards/rejected": 0.002333755139261484, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.04, |
| "grad_norm": 42.27700827156122, |
| "learning_rate": 2.7223230490018148e-08, |
| "logits/chosen": -1.6626800298690796, |
| "logits/rejected": -1.4588112831115723, |
| "logps/chosen": -84.01578521728516, |
| "logps/rejected": -111.53340911865234, |
| "loss": 0.6953, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.015042876824736595, |
| "rewards/margins": 0.0040522003546357155, |
| "rewards/rejected": 0.01099067647010088, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.04, |
| "eval_logits/chosen": -2.501953125, |
| "eval_logits/rejected": -2.2406537532806396, |
| "eval_logps/chosen": -99.84339141845703, |
| "eval_logps/rejected": -125.21623992919922, |
| "eval_loss": 0.6942028403282166, |
| "eval_rewards/accuracies": 0.5357142686843872, |
| "eval_rewards/chosen": 0.011287148110568523, |
| "eval_rewards/margins": -0.0042336308397352695, |
| "eval_rewards/rejected": 0.015520776621997356, |
| "eval_runtime": 73.7375, |
| "eval_samples_per_second": 12.07, |
| "eval_steps_per_second": 0.19, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.04, |
| "grad_norm": 46.94348163244785, |
| "learning_rate": 2.994555353901996e-08, |
| "logits/chosen": -1.4403226375579834, |
| "logits/rejected": -1.123073935508728, |
| "logps/chosen": -107.60018157958984, |
| "logps/rejected": -231.7578582763672, |
| "loss": 0.6892, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.009284459054470062, |
| "rewards/margins": 0.01384044624865055, |
| "rewards/rejected": -0.004555988125503063, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.04, |
| "grad_norm": 46.81740301639235, |
| "learning_rate": 3.266787658802178e-08, |
| "logits/chosen": -1.4798848628997803, |
| "logits/rejected": -1.3624470233917236, |
| "logps/chosen": -94.34761047363281, |
| "logps/rejected": -120.3208999633789, |
| "loss": 0.6925, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.016081811860203743, |
| "rewards/margins": 0.002866668626666069, |
| "rewards/rejected": 0.0132151423022151, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.05, |
| "grad_norm": 46.08791439590883, |
| "learning_rate": 3.539019963702359e-08, |
| "logits/chosen": -1.9895107746124268, |
| "logits/rejected": -1.281200885772705, |
| "logps/chosen": -95.02359771728516, |
| "logps/rejected": -131.6729736328125, |
| "loss": 0.6933, |
| "rewards/accuracies": 0.44999998807907104, |
| "rewards/chosen": 0.02468906342983246, |
| "rewards/margins": -0.0023543343413621187, |
| "rewards/rejected": 0.027043398469686508, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.05, |
| "grad_norm": 48.41164277694124, |
| "learning_rate": 3.81125226860254e-08, |
| "logits/chosen": -1.8839794397354126, |
| "logits/rejected": -1.5620959997177124, |
| "logps/chosen": -97.3235855102539, |
| "logps/rejected": -101.75816345214844, |
| "loss": 0.6896, |
| "rewards/accuracies": 0.25, |
| "rewards/chosen": 0.023149680346250534, |
| "rewards/margins": -0.027130965143442154, |
| "rewards/rejected": 0.050280649214982986, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.05, |
| "grad_norm": 42.39632204422603, |
| "learning_rate": 4.083484573502722e-08, |
| "logits/chosen": -1.9134677648544312, |
| "logits/rejected": -1.6838871240615845, |
| "logps/chosen": -72.68460845947266, |
| "logps/rejected": -104.17420959472656, |
| "loss": 0.6856, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.04995592683553696, |
| "rewards/margins": 0.012682494707405567, |
| "rewards/rejected": 0.03727342560887337, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.06, |
| "grad_norm": 45.64533725609479, |
| "learning_rate": 4.355716878402904e-08, |
| "logits/chosen": -1.3503320217132568, |
| "logits/rejected": -1.1550263166427612, |
| "logps/chosen": -110.38504791259766, |
| "logps/rejected": -130.46852111816406, |
| "loss": 0.69, |
| "rewards/accuracies": 0.44999998807907104, |
| "rewards/chosen": 0.05299491807818413, |
| "rewards/margins": -0.0019288047915324569, |
| "rewards/rejected": 0.05492372438311577, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.06, |
| "grad_norm": 41.18460830204607, |
| "learning_rate": 4.627949183303085e-08, |
| "logits/chosen": -1.496957778930664, |
| "logits/rejected": -0.9095319509506226, |
| "logps/chosen": -78.29255676269531, |
| "logps/rejected": -132.0447540283203, |
| "loss": 0.6846, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.059414975345134735, |
| "rewards/margins": 0.02102474495768547, |
| "rewards/rejected": 0.03839023411273956, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.07, |
| "grad_norm": 41.49152413703793, |
| "learning_rate": 4.9001814882032664e-08, |
| "logits/chosen": -1.5343455076217651, |
| "logits/rejected": -1.8757272958755493, |
| "logps/chosen": -103.69041442871094, |
| "logps/rejected": -111.36357116699219, |
| "loss": 0.687, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.07798389345407486, |
| "rewards/margins": 0.0013759995345026255, |
| "rewards/rejected": 0.07660789787769318, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.07, |
| "grad_norm": 42.39863370578702, |
| "learning_rate": 5.172413793103448e-08, |
| "logits/chosen": -1.6158069372177124, |
| "logits/rejected": -0.9124865531921387, |
| "logps/chosen": -100.64363861083984, |
| "logps/rejected": -135.06634521484375, |
| "loss": 0.6837, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.09579858928918839, |
| "rewards/margins": 0.01044096052646637, |
| "rewards/rejected": 0.08535762131214142, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.07, |
| "grad_norm": 39.9911934894391, |
| "learning_rate": 5.4446460980036295e-08, |
| "logits/chosen": -2.3459877967834473, |
| "logits/rejected": -1.5009344816207886, |
| "logps/chosen": -82.8287353515625, |
| "logps/rejected": -124.65399169921875, |
| "loss": 0.6799, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.14615079760551453, |
| "rewards/margins": -0.0005797408521175385, |
| "rewards/rejected": 0.14673054218292236, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.07, |
| "eval_logits/chosen": -2.5021491050720215, |
| "eval_logits/rejected": -2.2421486377716064, |
| "eval_logps/chosen": -98.44856262207031, |
| "eval_logps/rejected": -124.38037109375, |
| "eval_loss": 0.6807048916816711, |
| "eval_rewards/accuracies": 0.7857142686843872, |
| "eval_rewards/chosen": 0.15076985955238342, |
| "eval_rewards/margins": 0.05166209489107132, |
| "eval_rewards/rejected": 0.0991077572107315, |
| "eval_runtime": 71.9557, |
| "eval_samples_per_second": 12.369, |
| "eval_steps_per_second": 0.195, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 45.974517848209416, |
| "learning_rate": 5.716878402903811e-08, |
| "logits/chosen": -2.038114070892334, |
| "logits/rejected": -1.6292145252227783, |
| "logps/chosen": -95.0359115600586, |
| "logps/rejected": -117.90171813964844, |
| "loss": 0.6816, |
| "rewards/accuracies": 0.44999998807907104, |
| "rewards/chosen": 0.15096978843212128, |
| "rewards/margins": -0.013230574317276478, |
| "rewards/rejected": 0.16420036554336548, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 44.619784920786046, |
| "learning_rate": 5.989110707803992e-08, |
| "logits/chosen": -2.296483039855957, |
| "logits/rejected": -1.9715936183929443, |
| "logps/chosen": -93.15199279785156, |
| "logps/rejected": -127.34361267089844, |
| "loss": 0.674, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.20922093093395233, |
| "rewards/margins": 0.046642549335956573, |
| "rewards/rejected": 0.16257838904857635, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 44.88970750413813, |
| "learning_rate": 6.261343012704174e-08, |
| "logits/chosen": -1.450714349746704, |
| "logits/rejected": -0.9239526987075806, |
| "logps/chosen": -85.76863098144531, |
| "logps/rejected": -121.76054382324219, |
| "loss": 0.6702, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.1901402473449707, |
| "rewards/margins": 0.05737558752298355, |
| "rewards/rejected": 0.13276468217372894, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.09, |
| "grad_norm": 39.31008777722159, |
| "learning_rate": 6.533575317604356e-08, |
| "logits/chosen": -1.9880611896514893, |
| "logits/rejected": -1.3975508213043213, |
| "logps/chosen": -99.23193359375, |
| "logps/rejected": -116.69496154785156, |
| "loss": 0.6673, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": 0.2814417779445648, |
| "rewards/margins": 0.020699840039014816, |
| "rewards/rejected": 0.2607419490814209, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.09, |
| "grad_norm": 43.800148674203605, |
| "learning_rate": 6.805807622504536e-08, |
| "logits/chosen": -1.808105230331421, |
| "logits/rejected": -1.5273463726043701, |
| "logps/chosen": -86.12389373779297, |
| "logps/rejected": -122.4721908569336, |
| "loss": 0.666, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.2851946949958801, |
| "rewards/margins": 0.03660944104194641, |
| "rewards/rejected": 0.24858525395393372, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.09, |
| "grad_norm": 43.84613473195863, |
| "learning_rate": 7.078039927404718e-08, |
| "logits/chosen": -2.0416717529296875, |
| "logits/rejected": -1.812233567237854, |
| "logps/chosen": -118.04366302490234, |
| "logps/rejected": -118.3464126586914, |
| "loss": 0.6663, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.3446394205093384, |
| "rewards/margins": 0.07985617220401764, |
| "rewards/rejected": 0.26478323340415955, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.1, |
| "grad_norm": 45.46886488583846, |
| "learning_rate": 7.3502722323049e-08, |
| "logits/chosen": -2.089078664779663, |
| "logits/rejected": -1.5567834377288818, |
| "logps/chosen": -103.9489974975586, |
| "logps/rejected": -151.25582885742188, |
| "loss": 0.651, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.3747584819793701, |
| "rewards/margins": 0.04969606548547745, |
| "rewards/rejected": 0.32506245374679565, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.1, |
| "grad_norm": 46.872473628922336, |
| "learning_rate": 7.62250453720508e-08, |
| "logits/chosen": -1.9163455963134766, |
| "logits/rejected": -1.490391492843628, |
| "logps/chosen": -79.95561218261719, |
| "logps/rejected": -128.16030883789062, |
| "loss": 0.6558, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.5081700682640076, |
| "rewards/margins": 0.14635446667671204, |
| "rewards/rejected": 0.36181557178497314, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.11, |
| "grad_norm": 40.264154663026744, |
| "learning_rate": 7.894736842105262e-08, |
| "logits/chosen": -1.999222755432129, |
| "logits/rejected": -2.104238986968994, |
| "logps/chosen": -96.35908508300781, |
| "logps/rejected": -113.0312728881836, |
| "loss": 0.6477, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.603106677532196, |
| "rewards/margins": 0.18604376912117004, |
| "rewards/rejected": 0.4170629382133484, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.11, |
| "grad_norm": 38.393082680911135, |
| "learning_rate": 8.166969147005444e-08, |
| "logits/chosen": -2.011298894882202, |
| "logits/rejected": -1.8035573959350586, |
| "logps/chosen": -101.60588836669922, |
| "logps/rejected": -126.2813491821289, |
| "loss": 0.6328, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.5809694528579712, |
| "rewards/margins": 0.10704417526721954, |
| "rewards/rejected": 0.4739252030849457, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.11, |
| "eval_logits/chosen": -2.5139808654785156, |
| "eval_logits/rejected": -2.253553628921509, |
| "eval_logps/chosen": -93.76677703857422, |
| "eval_logps/rejected": -121.03182220458984, |
| "eval_loss": 0.6357866525650024, |
| "eval_rewards/accuracies": 0.8214285969734192, |
| "eval_rewards/chosen": 0.6189486384391785, |
| "eval_rewards/margins": 0.18498651683330536, |
| "eval_rewards/rejected": 0.4339621067047119, |
| "eval_runtime": 72.0138, |
| "eval_samples_per_second": 12.359, |
| "eval_steps_per_second": 0.194, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.11, |
| "grad_norm": 41.32014110656028, |
| "learning_rate": 8.439201451905626e-08, |
| "logits/chosen": -1.7809641361236572, |
| "logits/rejected": -1.4835566282272339, |
| "logps/chosen": -91.62686157226562, |
| "logps/rejected": -160.69992065429688, |
| "loss": 0.6213, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.6366099715232849, |
| "rewards/margins": 0.10506564378738403, |
| "rewards/rejected": 0.5315443277359009, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 40.20352730538307, |
| "learning_rate": 8.711433756805808e-08, |
| "logits/chosen": -1.3174822330474854, |
| "logits/rejected": -0.9151695370674133, |
| "logps/chosen": -76.4534912109375, |
| "logps/rejected": -106.75065612792969, |
| "loss": 0.6352, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.6975838541984558, |
| "rewards/margins": 0.11969652026891708, |
| "rewards/rejected": 0.5778872966766357, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 44.10982454650109, |
| "learning_rate": 8.983666061705989e-08, |
| "logits/chosen": -1.7297052145004272, |
| "logits/rejected": -1.1650458574295044, |
| "logps/chosen": -95.3366470336914, |
| "logps/rejected": -113.1480712890625, |
| "loss": 0.6209, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.8798357844352722, |
| "rewards/margins": 0.10148117691278458, |
| "rewards/rejected": 0.7783547043800354, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 39.48829529957203, |
| "learning_rate": 9.25589836660617e-08, |
| "logits/chosen": -1.7264535427093506, |
| "logits/rejected": -1.7544937133789062, |
| "logps/chosen": -100.7391586303711, |
| "logps/rejected": -132.95297241210938, |
| "loss": 0.6138, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.7276839017868042, |
| "rewards/margins": 0.1849050223827362, |
| "rewards/rejected": 0.5427789092063904, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.13, |
| "grad_norm": 40.75750118936868, |
| "learning_rate": 9.528130671506351e-08, |
| "logits/chosen": -1.9755557775497437, |
| "logits/rejected": -1.5455005168914795, |
| "logps/chosen": -92.50463104248047, |
| "logps/rejected": -114.50557708740234, |
| "loss": 0.6187, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.886059582233429, |
| "rewards/margins": 0.1289076954126358, |
| "rewards/rejected": 0.7571519613265991, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.13, |
| "grad_norm": 42.45802253147107, |
| "learning_rate": 9.800362976406533e-08, |
| "logits/chosen": -1.9283500909805298, |
| "logits/rejected": -1.5053136348724365, |
| "logps/chosen": -82.00525665283203, |
| "logps/rejected": -104.69712829589844, |
| "loss": 0.6127, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.1786202192306519, |
| "rewards/margins": 0.27130889892578125, |
| "rewards/rejected": 0.9073113203048706, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.13, |
| "grad_norm": 39.734876260743064, |
| "learning_rate": 1.0072595281306713e-07, |
| "logits/chosen": -2.2132327556610107, |
| "logits/rejected": -1.904916524887085, |
| "logps/chosen": -72.9496841430664, |
| "logps/rejected": -108.14723205566406, |
| "loss": 0.605, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 1.2401649951934814, |
| "rewards/margins": 0.1573180854320526, |
| "rewards/rejected": 1.0828468799591064, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.14, |
| "grad_norm": 39.83612602137503, |
| "learning_rate": 1.0344827586206897e-07, |
| "logits/chosen": -2.0371642112731934, |
| "logits/rejected": -1.9173485040664673, |
| "logps/chosen": -83.82426452636719, |
| "logps/rejected": -97.99388885498047, |
| "loss": 0.5913, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.2351922988891602, |
| "rewards/margins": 0.4066685140132904, |
| "rewards/rejected": 0.8285236358642578, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.14, |
| "grad_norm": 42.576323725278016, |
| "learning_rate": 1.0617059891107078e-07, |
| "logits/chosen": -2.1276021003723145, |
| "logits/rejected": -2.370884418487549, |
| "logps/chosen": -97.64468383789062, |
| "logps/rejected": -113.1034164428711, |
| "loss": 0.6069, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.331588864326477, |
| "rewards/margins": 0.19395257532596588, |
| "rewards/rejected": 1.1376360654830933, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.15, |
| "grad_norm": 44.1343637917746, |
| "learning_rate": 1.0889292196007259e-07, |
| "logits/chosen": -1.8790748119354248, |
| "logits/rejected": -1.5620168447494507, |
| "logps/chosen": -72.6387710571289, |
| "logps/rejected": -154.55137634277344, |
| "loss": 0.5964, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3714253902435303, |
| "rewards/margins": 0.4154202938079834, |
| "rewards/rejected": 0.9560050964355469, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.15, |
| "eval_logits/chosen": -2.5321178436279297, |
| "eval_logits/rejected": -2.2736294269561768, |
| "eval_logps/chosen": -86.19153594970703, |
| "eval_logps/rejected": -116.01555633544922, |
| "eval_loss": 0.5837011337280273, |
| "eval_rewards/accuracies": 0.7678571343421936, |
| "eval_rewards/chosen": 1.3764731884002686, |
| "eval_rewards/margins": 0.44088268280029297, |
| "eval_rewards/rejected": 0.9355906844139099, |
| "eval_runtime": 71.9235, |
| "eval_samples_per_second": 12.374, |
| "eval_steps_per_second": 0.195, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.15, |
| "grad_norm": 32.5866475732278, |
| "learning_rate": 1.1161524500907441e-07, |
| "logits/chosen": -2.1084635257720947, |
| "logits/rejected": -1.6532913446426392, |
| "logps/chosen": -71.08902740478516, |
| "logps/rejected": -119.5894546508789, |
| "loss": 0.5579, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.3634411096572876, |
| "rewards/margins": 0.15741710364818573, |
| "rewards/rejected": 1.206023931503296, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.15, |
| "grad_norm": 51.63908130905885, |
| "learning_rate": 1.1433756805807621e-07, |
| "logits/chosen": -1.8392841815948486, |
| "logits/rejected": -1.5305787324905396, |
| "logps/chosen": -64.91630554199219, |
| "logps/rejected": -90.24629974365234, |
| "loss": 0.5809, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 1.4629342555999756, |
| "rewards/margins": 0.21032755076885223, |
| "rewards/rejected": 1.2526066303253174, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 42.575288667126856, |
| "learning_rate": 1.1705989110707803e-07, |
| "logits/chosen": -1.7529628276824951, |
| "logits/rejected": -1.451038122177124, |
| "logps/chosen": -81.2251205444336, |
| "logps/rejected": -142.9680938720703, |
| "loss": 0.5776, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.41762375831604, |
| "rewards/margins": 0.20172078907489777, |
| "rewards/rejected": 1.2159030437469482, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 37.38028775120078, |
| "learning_rate": 1.1978221415607984e-07, |
| "logits/chosen": -1.7141106128692627, |
| "logits/rejected": -1.342930555343628, |
| "logps/chosen": -74.53529357910156, |
| "logps/rejected": -107.2362060546875, |
| "loss": 0.5626, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.7542121410369873, |
| "rewards/margins": 0.4075583517551422, |
| "rewards/rejected": 1.3466538190841675, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 36.24749956558924, |
| "learning_rate": 1.2250453720508167e-07, |
| "logits/chosen": -1.9711906909942627, |
| "logits/rejected": -1.221478819847107, |
| "logps/chosen": -103.87667083740234, |
| "logps/rejected": -135.00277709960938, |
| "loss": 0.5604, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.4585211277008057, |
| "rewards/margins": 0.28392547369003296, |
| "rewards/rejected": 1.1745957136154175, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.17, |
| "grad_norm": 42.96091232839455, |
| "learning_rate": 1.2522686025408348e-07, |
| "logits/chosen": -1.7234961986541748, |
| "logits/rejected": -1.5666788816452026, |
| "logps/chosen": -94.36329650878906, |
| "logps/rejected": -109.29872131347656, |
| "loss": 0.5283, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.7453796863555908, |
| "rewards/margins": 0.3029170632362366, |
| "rewards/rejected": 1.442462682723999, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.17, |
| "grad_norm": 48.636321311117, |
| "learning_rate": 1.279491833030853e-07, |
| "logits/chosen": -2.4974513053894043, |
| "logits/rejected": -1.580528736114502, |
| "logps/chosen": -93.00968170166016, |
| "logps/rejected": -127.61222076416016, |
| "loss": 0.5586, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.8251793384552002, |
| "rewards/margins": 0.470356285572052, |
| "rewards/rejected": 1.354823112487793, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.17, |
| "grad_norm": 35.14478185239256, |
| "learning_rate": 1.3067150635208711e-07, |
| "logits/chosen": -1.878103256225586, |
| "logits/rejected": -1.7810052633285522, |
| "logps/chosen": -64.92475891113281, |
| "logps/rejected": -95.4583511352539, |
| "loss": 0.5455, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.9156177043914795, |
| "rewards/margins": 0.21928730607032776, |
| "rewards/rejected": 1.6963306665420532, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.18, |
| "grad_norm": 41.07425605129, |
| "learning_rate": 1.3339382940108892e-07, |
| "logits/chosen": -1.8308073282241821, |
| "logits/rejected": -1.6511573791503906, |
| "logps/chosen": -94.47706604003906, |
| "logps/rejected": -142.56326293945312, |
| "loss": 0.5669, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.7191970348358154, |
| "rewards/margins": 0.4305998682975769, |
| "rewards/rejected": 1.2885972261428833, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.18, |
| "grad_norm": 40.72644317271578, |
| "learning_rate": 1.3611615245009072e-07, |
| "logits/chosen": -1.8288648128509521, |
| "logits/rejected": -1.6132848262786865, |
| "logps/chosen": -84.89088439941406, |
| "logps/rejected": -141.57489013671875, |
| "loss": 0.5567, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.799727201461792, |
| "rewards/margins": 0.7182740569114685, |
| "rewards/rejected": 1.0814530849456787, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.18, |
| "eval_logits/chosen": -2.56964373588562, |
| "eval_logits/rejected": -2.3121345043182373, |
| "eval_logps/chosen": -81.02156829833984, |
| "eval_logps/rejected": -113.64513397216797, |
| "eval_loss": 0.5355415940284729, |
| "eval_rewards/accuracies": 0.8392857313156128, |
| "eval_rewards/chosen": 1.8934696912765503, |
| "eval_rewards/margins": 0.7208380699157715, |
| "eval_rewards/rejected": 1.1726313829421997, |
| "eval_runtime": 71.9101, |
| "eval_samples_per_second": 12.377, |
| "eval_steps_per_second": 0.195, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.19, |
| "grad_norm": 31.50812715169961, |
| "learning_rate": 1.3883847549909256e-07, |
| "logits/chosen": -2.6629555225372314, |
| "logits/rejected": -1.8640201091766357, |
| "logps/chosen": -64.94083404541016, |
| "logps/rejected": -131.4000701904297, |
| "loss": 0.5287, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 2.017392635345459, |
| "rewards/margins": 0.5965152978897095, |
| "rewards/rejected": 1.4208776950836182, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.19, |
| "grad_norm": 41.001026190013995, |
| "learning_rate": 1.4156079854809436e-07, |
| "logits/chosen": -2.0135555267333984, |
| "logits/rejected": -2.3467681407928467, |
| "logps/chosen": -83.2616958618164, |
| "logps/rejected": -96.27758026123047, |
| "loss": 0.5086, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.7922933101654053, |
| "rewards/margins": 0.6331424117088318, |
| "rewards/rejected": 1.1591508388519287, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.19, |
| "grad_norm": 43.50803723743233, |
| "learning_rate": 1.442831215970962e-07, |
| "logits/chosen": -2.3700008392333984, |
| "logits/rejected": -1.9721641540527344, |
| "logps/chosen": -65.375, |
| "logps/rejected": -107.3283920288086, |
| "loss": 0.5664, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 1.932377576828003, |
| "rewards/margins": 0.5669600367546082, |
| "rewards/rejected": 1.36541748046875, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 36.6257172454756, |
| "learning_rate": 1.47005444646098e-07, |
| "logits/chosen": -2.7627670764923096, |
| "logits/rejected": -2.018375873565674, |
| "logps/chosen": -66.04200744628906, |
| "logps/rejected": -90.71024322509766, |
| "loss": 0.5114, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.1180334091186523, |
| "rewards/margins": 0.621078610420227, |
| "rewards/rejected": 1.4969549179077148, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 35.991193860058296, |
| "learning_rate": 1.497277676950998e-07, |
| "logits/chosen": -1.3566482067108154, |
| "logits/rejected": -1.3935582637786865, |
| "logps/chosen": -77.91327667236328, |
| "logps/rejected": -97.44300079345703, |
| "loss": 0.5308, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.9388548135757446, |
| "rewards/margins": 0.38032227754592896, |
| "rewards/rejected": 1.5585325956344604, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 48.06228927830701, |
| "learning_rate": 1.49998780935628e-07, |
| "logits/chosen": -1.7465827465057373, |
| "logits/rejected": -1.4473216533660889, |
| "logps/chosen": -70.13250732421875, |
| "logps/rejected": -105.08538818359375, |
| "loss": 0.4779, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5702747106552124, |
| "rewards/margins": 0.8154007196426392, |
| "rewards/rejected": 0.7548739910125732, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.21, |
| "grad_norm": 38.75394797505258, |
| "learning_rate": 1.4999456693682575e-07, |
| "logits/chosen": -2.4446985721588135, |
| "logits/rejected": -1.8925511837005615, |
| "logps/chosen": -71.6372299194336, |
| "logps/rejected": -103.564697265625, |
| "loss": 0.5129, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 2.205054759979248, |
| "rewards/margins": 0.5576547384262085, |
| "rewards/rejected": 1.647400140762329, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.21, |
| "grad_norm": 37.78034263492822, |
| "learning_rate": 1.4998734312249918e-07, |
| "logits/chosen": -1.8992903232574463, |
| "logits/rejected": -1.5952703952789307, |
| "logps/chosen": -83.2913589477539, |
| "logps/rejected": -120.90938568115234, |
| "loss": 0.4891, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.9770288467407227, |
| "rewards/margins": 0.7018577456474304, |
| "rewards/rejected": 1.2751710414886475, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.21, |
| "grad_norm": 37.211325491550305, |
| "learning_rate": 1.4997710978256735e-07, |
| "logits/chosen": -1.94363272190094, |
| "logits/rejected": -1.9168386459350586, |
| "logps/chosen": -76.82261657714844, |
| "logps/rejected": -120.50138092041016, |
| "loss": 0.5175, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.0862905979156494, |
| "rewards/margins": 0.5364335775375366, |
| "rewards/rejected": 1.5498571395874023, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.22, |
| "grad_norm": 38.42991397247903, |
| "learning_rate": 1.4996386732773285e-07, |
| "logits/chosen": -1.6784331798553467, |
| "logits/rejected": -1.4212602376937866, |
| "logps/chosen": -78.51207733154297, |
| "logps/rejected": -97.93910217285156, |
| "loss": 0.5234, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.9482488632202148, |
| "rewards/margins": 0.5499599575996399, |
| "rewards/rejected": 1.3982888460159302, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.22, |
| "eval_logits/chosen": -2.631465435028076, |
| "eval_logits/rejected": -2.3776392936706543, |
| "eval_logps/chosen": -79.7362289428711, |
| "eval_logps/rejected": -114.89885711669922, |
| "eval_loss": 0.4917795658111572, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 2.022002696990967, |
| "eval_rewards/margins": 0.9747439026832581, |
| "eval_rewards/rejected": 1.047258734703064, |
| "eval_runtime": 71.9087, |
| "eval_samples_per_second": 12.377, |
| "eval_steps_per_second": 0.195, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.22, |
| "grad_norm": 48.91223436910098, |
| "learning_rate": 1.4994761628946546e-07, |
| "logits/chosen": -1.6232852935791016, |
| "logits/rejected": -1.4054292440414429, |
| "logps/chosen": -76.1790542602539, |
| "logps/rejected": -125.78961181640625, |
| "loss": 0.4929, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.940302848815918, |
| "rewards/margins": 0.9549075961112976, |
| "rewards/rejected": 0.9853953123092651, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.23, |
| "grad_norm": 41.70252788830195, |
| "learning_rate": 1.499283573199808e-07, |
| "logits/chosen": -1.7214891910552979, |
| "logits/rejected": -1.2875360250473022, |
| "logps/chosen": -73.24628448486328, |
| "logps/rejected": -86.8110122680664, |
| "loss": 0.4935, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.8633216619491577, |
| "rewards/margins": 0.7748242020606995, |
| "rewards/rejected": 1.0884974002838135, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.23, |
| "grad_norm": 38.30334820196022, |
| "learning_rate": 1.499060911922141e-07, |
| "logits/chosen": -1.616151213645935, |
| "logits/rejected": -1.6390151977539062, |
| "logps/chosen": -68.93297576904297, |
| "logps/rejected": -113.6138687133789, |
| "loss": 0.4795, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.810837984085083, |
| "rewards/margins": 0.38780540227890015, |
| "rewards/rejected": 1.423032522201538, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.23, |
| "grad_norm": 34.79521166345824, |
| "learning_rate": 1.498808187997893e-07, |
| "logits/chosen": -1.9353692531585693, |
| "logits/rejected": -1.5537188053131104, |
| "logps/chosen": -97.22018432617188, |
| "logps/rejected": -153.95028686523438, |
| "loss": 0.5006, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.8405921459197998, |
| "rewards/margins": 1.0561001300811768, |
| "rewards/rejected": 0.7844920754432678, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 94.46732088298053, |
| "learning_rate": 1.4985254115698304e-07, |
| "logits/chosen": -2.263124704360962, |
| "logits/rejected": -1.8345104455947876, |
| "logps/chosen": -73.39897155761719, |
| "logps/rejected": -92.21670532226562, |
| "loss": 0.4917, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.1108241081237793, |
| "rewards/margins": 0.9142869114875793, |
| "rewards/rejected": 1.1965371370315552, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 33.786235958694014, |
| "learning_rate": 1.4982125939868402e-07, |
| "logits/chosen": -2.4463446140289307, |
| "logits/rejected": -1.6779578924179077, |
| "logps/chosen": -79.39064025878906, |
| "logps/rejected": -132.6228790283203, |
| "loss": 0.4479, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.3004369735717773, |
| "rewards/margins": 1.3063924312591553, |
| "rewards/rejected": 0.9940446019172668, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 35.539345812176975, |
| "learning_rate": 1.4978697478034753e-07, |
| "logits/chosen": -2.2131102085113525, |
| "logits/rejected": -1.8831088542938232, |
| "logps/chosen": -67.44413757324219, |
| "logps/rejected": -113.38688659667969, |
| "loss": 0.4648, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.094628095626831, |
| "rewards/margins": 0.9700366854667664, |
| "rewards/rejected": 1.124591588973999, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.25, |
| "grad_norm": 38.83509762475066, |
| "learning_rate": 1.497496886779449e-07, |
| "logits/chosen": -2.4378554821014404, |
| "logits/rejected": -1.9790445566177368, |
| "logps/chosen": -103.16511535644531, |
| "logps/rejected": -128.8167266845703, |
| "loss": 0.4337, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.8627456426620483, |
| "rewards/margins": 0.8879152536392212, |
| "rewards/rejected": 0.9748304486274719, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.25, |
| "grad_norm": 41.327033787846, |
| "learning_rate": 1.497094025879084e-07, |
| "logits/chosen": -1.8331416845321655, |
| "logits/rejected": -1.4264384508132935, |
| "logps/chosen": -80.54314422607422, |
| "logps/rejected": -137.10955810546875, |
| "loss": 0.4143, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.4691455364227295, |
| "rewards/margins": 0.8567919731140137, |
| "rewards/rejected": 0.612353503704071, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.25, |
| "grad_norm": 38.09387686336411, |
| "learning_rate": 1.4966611812707116e-07, |
| "logits/chosen": -2.376185894012451, |
| "logits/rejected": -1.9318294525146484, |
| "logps/chosen": -67.70744323730469, |
| "logps/rejected": -106.6443099975586, |
| "loss": 0.4468, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 2.5539474487304688, |
| "rewards/margins": 1.1218903064727783, |
| "rewards/rejected": 1.43205726146698, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.25, |
| "eval_logits/chosen": -2.6510469913482666, |
| "eval_logits/rejected": -2.410930871963501, |
| "eval_logps/chosen": -79.01429748535156, |
| "eval_logps/rejected": -117.04235076904297, |
| "eval_loss": 0.45572710037231445, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 2.0941965579986572, |
| "eval_rewards/margins": 1.2612864971160889, |
| "eval_rewards/rejected": 0.832909882068634, |
| "eval_runtime": 72.043, |
| "eval_samples_per_second": 12.354, |
| "eval_steps_per_second": 0.194, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.26, |
| "grad_norm": 34.41497827448044, |
| "learning_rate": 1.4961983703260222e-07, |
| "logits/chosen": -2.285163402557373, |
| "logits/rejected": -1.6250555515289307, |
| "logps/chosen": -82.45291137695312, |
| "logps/rejected": -162.6754913330078, |
| "loss": 0.4388, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.6247352361679077, |
| "rewards/margins": 1.2791966199874878, |
| "rewards/rejected": 0.34553852677345276, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.26, |
| "grad_norm": 38.939975049466675, |
| "learning_rate": 1.495705611619369e-07, |
| "logits/chosen": -2.8705544471740723, |
| "logits/rejected": -1.9211571216583252, |
| "logps/chosen": -52.576690673828125, |
| "logps/rejected": -104.49955749511719, |
| "loss": 0.4632, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.5734798908233643, |
| "rewards/margins": 1.8182973861694336, |
| "rewards/rejected": 0.755182683467865, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.26, |
| "grad_norm": 37.237058738428075, |
| "learning_rate": 1.4951829249270223e-07, |
| "logits/chosen": -2.2160234451293945, |
| "logits/rejected": -1.7666336297988892, |
| "logps/chosen": -94.98374938964844, |
| "logps/rejected": -155.47586059570312, |
| "loss": 0.4477, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.8300750255584717, |
| "rewards/margins": 0.9866692423820496, |
| "rewards/rejected": 0.8434059023857117, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.27, |
| "grad_norm": 42.247815955525425, |
| "learning_rate": 1.4946303312263751e-07, |
| "logits/chosen": -2.1757843494415283, |
| "logits/rejected": -1.8440332412719727, |
| "logps/chosen": -72.95508575439453, |
| "logps/rejected": -113.18672943115234, |
| "loss": 0.4715, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 2.1009631156921387, |
| "rewards/margins": 1.115952491760254, |
| "rewards/rejected": 0.9850105047225952, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.27, |
| "grad_norm": 32.89732640498844, |
| "learning_rate": 1.4940478526951018e-07, |
| "logits/chosen": -1.856689453125, |
| "logits/rejected": -1.4313759803771973, |
| "logps/chosen": -82.71755981445312, |
| "logps/rejected": -135.11648559570312, |
| "loss": 0.426, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.647456407546997, |
| "rewards/margins": 1.2215721607208252, |
| "rewards/rejected": 0.425883948802948, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.28, |
| "grad_norm": 39.76459644604554, |
| "learning_rate": 1.4934355127102686e-07, |
| "logits/chosen": -2.3893539905548096, |
| "logits/rejected": -1.5895264148712158, |
| "logps/chosen": -78.76719665527344, |
| "logps/rejected": -160.42080688476562, |
| "loss": 0.4394, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.3389594554901123, |
| "rewards/margins": 1.721954345703125, |
| "rewards/rejected": 0.6170052289962769, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.28, |
| "grad_norm": 30.768392184975625, |
| "learning_rate": 1.492793335847394e-07, |
| "logits/chosen": -2.251796245574951, |
| "logits/rejected": -1.5280625820159912, |
| "logps/chosen": -63.670433044433594, |
| "logps/rejected": -124.48072814941406, |
| "loss": 0.4001, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.670900583267212, |
| "rewards/margins": 1.3456604480743408, |
| "rewards/rejected": 0.3252400755882263, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.28, |
| "grad_norm": 35.26526799612695, |
| "learning_rate": 1.4921213478794637e-07, |
| "logits/chosen": -2.0840353965759277, |
| "logits/rejected": -1.9413776397705078, |
| "logps/chosen": -73.97483825683594, |
| "logps/rejected": -113.29158020019531, |
| "loss": 0.4185, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.834567666053772, |
| "rewards/margins": 1.0754048824310303, |
| "rewards/rejected": 0.7591627240180969, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.29, |
| "grad_norm": 42.37614388323698, |
| "learning_rate": 1.4914195757758955e-07, |
| "logits/chosen": -2.1000962257385254, |
| "logits/rejected": -2.3834102153778076, |
| "logps/chosen": -76.1771240234375, |
| "logps/rejected": -88.94342803955078, |
| "loss": 0.4397, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.1757988929748535, |
| "rewards/margins": 1.1148507595062256, |
| "rewards/rejected": 1.060948133468628, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.29, |
| "grad_norm": 37.69381725528037, |
| "learning_rate": 1.4906880477014573e-07, |
| "logits/chosen": -2.2119500637054443, |
| "logits/rejected": -1.4499582052230835, |
| "logps/chosen": -62.1577033996582, |
| "logps/rejected": -163.81442260742188, |
| "loss": 0.4445, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.9283027648925781, |
| "rewards/margins": 1.5437467098236084, |
| "rewards/rejected": 0.3845560550689697, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.29, |
| "eval_logits/chosen": -2.6051876544952393, |
| "eval_logits/rejected": -2.3914036750793457, |
| "eval_logps/chosen": -83.42094421386719, |
| "eval_logps/rejected": -123.46272277832031, |
| "eval_loss": 0.43243297934532166, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.6535331010818481, |
| "eval_rewards/margins": 1.4626604318618774, |
| "eval_rewards/rejected": 0.1908726841211319, |
| "eval_runtime": 71.9896, |
| "eval_samples_per_second": 12.363, |
| "eval_steps_per_second": 0.194, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.29, |
| "grad_norm": 35.55693000649927, |
| "learning_rate": 1.489926793015137e-07, |
| "logits/chosen": -2.137413501739502, |
| "logits/rejected": -1.9606491327285767, |
| "logps/chosen": -69.8790512084961, |
| "logps/rejected": -97.57211303710938, |
| "loss": 0.3881, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.0780739784240723, |
| "rewards/margins": 1.4736759662628174, |
| "rewards/rejected": 0.6043978333473206, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.3, |
| "grad_norm": 37.02292806830243, |
| "learning_rate": 1.489135842268963e-07, |
| "logits/chosen": -2.0538628101348877, |
| "logits/rejected": -2.0753753185272217, |
| "logps/chosen": -106.13938903808594, |
| "logps/rejected": -144.13514709472656, |
| "loss": 0.4145, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.3868672847747803, |
| "rewards/margins": 1.0619122982025146, |
| "rewards/rejected": 0.3249548673629761, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.3, |
| "grad_norm": 40.671728265294014, |
| "learning_rate": 1.4883152272067798e-07, |
| "logits/chosen": -1.9181344509124756, |
| "logits/rejected": -1.6507272720336914, |
| "logps/chosen": -99.54740905761719, |
| "logps/rejected": -119.50572204589844, |
| "loss": 0.4404, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.4642399549484253, |
| "rewards/margins": 0.9573423266410828, |
| "rewards/rejected": 0.5068976283073425, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.3, |
| "grad_norm": 28.025590997376412, |
| "learning_rate": 1.487464980762972e-07, |
| "logits/chosen": -2.1211495399475098, |
| "logits/rejected": -1.789926528930664, |
| "logps/chosen": -76.32536315917969, |
| "logps/rejected": -180.9839324951172, |
| "loss": 0.393, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8189208507537842, |
| "rewards/margins": 1.659450888633728, |
| "rewards/rejected": 0.15946999192237854, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.31, |
| "grad_norm": 42.494743726734264, |
| "learning_rate": 1.4865851370611445e-07, |
| "logits/chosen": -1.7717339992523193, |
| "logits/rejected": -1.7804222106933594, |
| "logps/chosen": -94.37425994873047, |
| "logps/rejected": -134.35830688476562, |
| "loss": 0.3825, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.593718409538269, |
| "rewards/margins": 1.334201455116272, |
| "rewards/rejected": 0.25951701402664185, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.31, |
| "grad_norm": 47.60353919509746, |
| "learning_rate": 1.4856757314127514e-07, |
| "logits/chosen": -1.930068016052246, |
| "logits/rejected": -1.686745285987854, |
| "logps/chosen": -87.13040161132812, |
| "logps/rejected": -108.21986389160156, |
| "loss": 0.3977, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.576547622680664, |
| "rewards/margins": 1.2103341817855835, |
| "rewards/rejected": 0.3662133812904358, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 55.2942123107171, |
| "learning_rate": 1.4847368003156803e-07, |
| "logits/chosen": -1.7101478576660156, |
| "logits/rejected": -1.5123283863067627, |
| "logps/chosen": -80.416259765625, |
| "logps/rejected": -92.58718872070312, |
| "loss": 0.4117, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.8133132457733154, |
| "rewards/margins": 1.0088872909545898, |
| "rewards/rejected": 0.8044260144233704, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 35.816541721556696, |
| "learning_rate": 1.483768381452786e-07, |
| "logits/chosen": -2.319746732711792, |
| "logits/rejected": -1.79997980594635, |
| "logps/chosen": -87.76374816894531, |
| "logps/rejected": -112.37357330322266, |
| "loss": 0.3523, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.6848220825195312, |
| "rewards/margins": 0.9358736276626587, |
| "rewards/rejected": 0.7489483952522278, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 36.93257214414797, |
| "learning_rate": 1.482770513690379e-07, |
| "logits/chosen": -1.800686240196228, |
| "logits/rejected": -1.3662943840026855, |
| "logps/chosen": -88.72349548339844, |
| "logps/rejected": -132.239501953125, |
| "loss": 0.3748, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5943005084991455, |
| "rewards/margins": 1.0010852813720703, |
| "rewards/rejected": 0.5932152271270752, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.33, |
| "grad_norm": 32.655789909757885, |
| "learning_rate": 1.4817432370766656e-07, |
| "logits/chosen": -1.7111234664916992, |
| "logits/rejected": -1.62754225730896, |
| "logps/chosen": -72.77391052246094, |
| "logps/rejected": -117.9988021850586, |
| "loss": 0.4219, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.0136775970458984, |
| "rewards/margins": 1.9716641902923584, |
| "rewards/rejected": 0.042013369500637054, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.33, |
| "eval_logits/chosen": -2.572319507598877, |
| "eval_logits/rejected": -2.3702971935272217, |
| "eval_logps/chosen": -83.57896423339844, |
| "eval_logps/rejected": -125.93928527832031, |
| "eval_loss": 0.4064531624317169, |
| "eval_rewards/accuracies": 0.9107142686843872, |
| "eval_rewards/chosen": 1.6377297639846802, |
| "eval_rewards/margins": 1.694512963294983, |
| "eval_rewards/rejected": -0.056782953441143036, |
| "eval_runtime": 71.9888, |
| "eval_samples_per_second": 12.363, |
| "eval_steps_per_second": 0.194, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.33, |
| "grad_norm": 34.63562171188158, |
| "learning_rate": 1.4806865928401402e-07, |
| "logits/chosen": -2.034156084060669, |
| "logits/rejected": -1.3545467853546143, |
| "logps/chosen": -63.69415283203125, |
| "logps/rejected": -122.93882751464844, |
| "loss": 0.3752, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.400649309158325, |
| "rewards/margins": 2.036404609680176, |
| "rewards/rejected": 0.36424484848976135, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.33, |
| "grad_norm": 51.406692521659814, |
| "learning_rate": 1.4796006233879314e-07, |
| "logits/chosen": -1.6481889486312866, |
| "logits/rejected": -1.8775209188461304, |
| "logps/chosen": -105.07774353027344, |
| "logps/rejected": -128.53115844726562, |
| "loss": 0.4173, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.0885381698608398, |
| "rewards/margins": 0.976910412311554, |
| "rewards/rejected": 0.11162771284580231, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.34, |
| "grad_norm": 36.79775770964529, |
| "learning_rate": 1.4784853723040993e-07, |
| "logits/chosen": -1.9353240728378296, |
| "logits/rejected": -1.7171472311019897, |
| "logps/chosen": -96.37171173095703, |
| "logps/rejected": -142.31790161132812, |
| "loss": 0.4281, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.0895094871520996, |
| "rewards/margins": 1.8335977792739868, |
| "rewards/rejected": -0.7440882921218872, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.34, |
| "grad_norm": 41.812337440587434, |
| "learning_rate": 1.4773408843478865e-07, |
| "logits/chosen": -2.2208828926086426, |
| "logits/rejected": -1.9805580377578735, |
| "logps/chosen": -97.20782470703125, |
| "logps/rejected": -128.40142822265625, |
| "loss": 0.3873, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3034839630126953, |
| "rewards/margins": 1.4772690534591675, |
| "rewards/rejected": -0.17378509044647217, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.34, |
| "grad_norm": 42.4893277983641, |
| "learning_rate": 1.4761672054519223e-07, |
| "logits/chosen": -2.1439785957336426, |
| "logits/rejected": -2.007866144180298, |
| "logps/chosen": -98.85569763183594, |
| "logps/rejected": -139.9319305419922, |
| "loss": 0.3826, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.8108327388763428, |
| "rewards/margins": 1.2574350833892822, |
| "rewards/rejected": 0.5533978343009949, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.35, |
| "grad_norm": 33.46468899519379, |
| "learning_rate": 1.4749643827203783e-07, |
| "logits/chosen": -1.9708385467529297, |
| "logits/rejected": -1.5546438694000244, |
| "logps/chosen": -93.60973358154297, |
| "logps/rejected": -142.23947143554688, |
| "loss": 0.3418, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.3536150455474854, |
| "rewards/margins": 1.6791467666625977, |
| "rewards/rejected": -0.32553163170814514, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.35, |
| "grad_norm": 37.27542798597497, |
| "learning_rate": 1.4737324644270786e-07, |
| "logits/chosen": -2.059234142303467, |
| "logits/rejected": -2.1984705924987793, |
| "logps/chosen": -100.48094177246094, |
| "logps/rejected": -111.93769836425781, |
| "loss": 0.4226, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.6639249324798584, |
| "rewards/margins": 1.1406528949737549, |
| "rewards/rejected": 0.5232721567153931, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 41.082132677027026, |
| "learning_rate": 1.4724715000135616e-07, |
| "logits/chosen": -1.8186811208724976, |
| "logits/rejected": -1.4485307931900024, |
| "logps/chosen": -99.40641784667969, |
| "logps/rejected": -124.72676086425781, |
| "loss": 0.4518, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.4649403095245361, |
| "rewards/margins": 1.2863953113555908, |
| "rewards/rejected": 0.1785450279712677, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 45.98733730441951, |
| "learning_rate": 1.4711815400870976e-07, |
| "logits/chosen": -1.4492137432098389, |
| "logits/rejected": -1.3497685194015503, |
| "logps/chosen": -80.22025299072266, |
| "logps/rejected": -121.85737609863281, |
| "loss": 0.3912, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.7067760229110718, |
| "rewards/margins": 2.1301562786102295, |
| "rewards/rejected": -0.42338013648986816, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 38.58236345723491, |
| "learning_rate": 1.4698626364186557e-07, |
| "logits/chosen": -2.125136137008667, |
| "logits/rejected": -1.753357172012329, |
| "logps/chosen": -95.4744644165039, |
| "logps/rejected": -130.9934539794922, |
| "loss": 0.4453, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.6447858810424805, |
| "rewards/margins": 1.4352115392684937, |
| "rewards/rejected": 0.2095741331577301, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.36, |
| "eval_logits/chosen": -2.6885669231414795, |
| "eval_logits/rejected": -2.4480230808258057, |
| "eval_logps/chosen": -84.5037841796875, |
| "eval_logps/rejected": -127.53032684326172, |
| "eval_loss": 0.38953348994255066, |
| "eval_rewards/accuracies": 0.9285714030265808, |
| "eval_rewards/chosen": 1.545248031616211, |
| "eval_rewards/margins": 1.7611348628997803, |
| "eval_rewards/rejected": -0.2158866971731186, |
| "eval_runtime": 71.9853, |
| "eval_samples_per_second": 12.364, |
| "eval_steps_per_second": 0.194, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.37, |
| "grad_norm": 37.010975459317685, |
| "learning_rate": 1.4685148419408265e-07, |
| "logits/chosen": -2.1752285957336426, |
| "logits/rejected": -1.78762686252594, |
| "logps/chosen": -85.75431060791016, |
| "logps/rejected": -120.37774658203125, |
| "loss": 0.3768, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.1561453342437744, |
| "rewards/margins": 1.0167858600616455, |
| "rewards/rejected": 0.1393592357635498, |
| "step": 1010 |
| }, |
| { |
| "epoch": 0.37, |
| "grad_norm": 35.576239317002795, |
| "learning_rate": 1.4671382107456988e-07, |
| "logits/chosen": -1.7128040790557861, |
| "logits/rejected": -1.5929228067398071, |
| "logps/chosen": -87.97055053710938, |
| "logps/rejected": -114.09513092041016, |
| "loss": 0.363, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.064514398574829, |
| "rewards/margins": 0.6555222272872925, |
| "rewards/rejected": 0.408992201089859, |
| "step": 1020 |
| }, |
| { |
| "epoch": 0.37, |
| "grad_norm": 48.8875893207821, |
| "learning_rate": 1.465732798082687e-07, |
| "logits/chosen": -1.8439133167266846, |
| "logits/rejected": -1.883114218711853, |
| "logps/chosen": -83.29924011230469, |
| "logps/rejected": -107.3209228515625, |
| "loss": 0.3854, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5590392351150513, |
| "rewards/margins": 1.844124436378479, |
| "rewards/rejected": -0.2850852608680725, |
| "step": 1030 |
| }, |
| { |
| "epoch": 0.38, |
| "grad_norm": 37.219215466416436, |
| "learning_rate": 1.4642986603563156e-07, |
| "logits/chosen": -2.579458713531494, |
| "logits/rejected": -2.0807945728302, |
| "logps/chosen": -63.25598907470703, |
| "logps/rejected": -125.60914611816406, |
| "loss": 0.39, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.9264271259307861, |
| "rewards/margins": 1.684195876121521, |
| "rewards/rejected": 0.2422313243150711, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.38, |
| "grad_norm": 34.650758595758035, |
| "learning_rate": 1.4628358551239537e-07, |
| "logits/chosen": -2.4567530155181885, |
| "logits/rejected": -1.7575896978378296, |
| "logps/chosen": -67.78668975830078, |
| "logps/rejected": -145.57591247558594, |
| "loss": 0.3579, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 2.0559427738189697, |
| "rewards/margins": 1.9872772693634033, |
| "rewards/rejected": 0.06866538524627686, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.38, |
| "grad_norm": 28.530221009217506, |
| "learning_rate": 1.461344441093506e-07, |
| "logits/chosen": -1.84628164768219, |
| "logits/rejected": -1.4732797145843506, |
| "logps/chosen": -92.16481018066406, |
| "logps/rejected": -120.20820617675781, |
| "loss": 0.3983, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.2543227672576904, |
| "rewards/margins": 0.7427952885627747, |
| "rewards/rejected": 0.5115275979042053, |
| "step": 1060 |
| }, |
| { |
| "epoch": 0.39, |
| "grad_norm": 39.23433854659703, |
| "learning_rate": 1.4598244781210573e-07, |
| "logits/chosen": -2.2004318237304688, |
| "logits/rejected": -1.6184364557266235, |
| "logps/chosen": -83.8439712524414, |
| "logps/rejected": -131.82962036132812, |
| "loss": 0.3871, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4584438800811768, |
| "rewards/margins": 1.8565394878387451, |
| "rewards/rejected": -0.39809566736221313, |
| "step": 1070 |
| }, |
| { |
| "epoch": 0.39, |
| "grad_norm": 35.45066510618733, |
| "learning_rate": 1.4582760272084676e-07, |
| "logits/chosen": -1.8293546438217163, |
| "logits/rejected": -1.3871347904205322, |
| "logps/chosen": -76.8940658569336, |
| "logps/rejected": -131.3907470703125, |
| "loss": 0.3741, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.866342306137085, |
| "rewards/margins": 1.9369169473648071, |
| "rewards/rejected": -0.07057473063468933, |
| "step": 1080 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 37.751293634401364, |
| "learning_rate": 1.4566991505009272e-07, |
| "logits/chosen": -2.4931588172912598, |
| "logits/rejected": -2.37707781791687, |
| "logps/chosen": -91.82124328613281, |
| "logps/rejected": -113.77565002441406, |
| "loss": 0.3502, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.9339624643325806, |
| "rewards/margins": 0.9942086338996887, |
| "rewards/rejected": 0.9397537112236023, |
| "step": 1090 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 68.79835073894007, |
| "learning_rate": 1.4550939112844606e-07, |
| "logits/chosen": -2.0712924003601074, |
| "logits/rejected": -1.6202366352081299, |
| "logps/chosen": -62.410888671875, |
| "logps/rejected": -119.64176940917969, |
| "loss": 0.372, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.7587225437164307, |
| "rewards/margins": 2.3271777629852295, |
| "rewards/rejected": -0.5684553980827332, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.4, |
| "eval_logits/chosen": -2.6049461364746094, |
| "eval_logits/rejected": -2.3804662227630615, |
| "eval_logps/chosen": -83.14766693115234, |
| "eval_logps/rejected": -129.16748046875, |
| "eval_loss": 0.373942106962204, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.6808584928512573, |
| "eval_rewards/margins": 2.0604610443115234, |
| "eval_rewards/rejected": -0.37960249185562134, |
| "eval_runtime": 71.9703, |
| "eval_samples_per_second": 12.366, |
| "eval_steps_per_second": 0.195, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 49.198262614504216, |
| "learning_rate": 1.453460373983387e-07, |
| "logits/chosen": -2.2133469581604004, |
| "logits/rejected": -1.7755333185195923, |
| "logps/chosen": -86.2526626586914, |
| "logps/rejected": -125.26475524902344, |
| "loss": 0.4013, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4716092348098755, |
| "rewards/margins": 2.0779099464416504, |
| "rewards/rejected": -0.606300950050354, |
| "step": 1110 |
| }, |
| { |
| "epoch": 0.41, |
| "grad_norm": 26.816352398316827, |
| "learning_rate": 1.451798604157734e-07, |
| "logits/chosen": -2.0376839637756348, |
| "logits/rejected": -1.5721218585968018, |
| "logps/chosen": -62.28696823120117, |
| "logps/rejected": -122.70638275146484, |
| "loss": 0.3371, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.9331592321395874, |
| "rewards/margins": 1.9306316375732422, |
| "rewards/rejected": 0.0025276304222643375, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.41, |
| "grad_norm": 30.305040452259234, |
| "learning_rate": 1.4501086685006087e-07, |
| "logits/chosen": -2.2330105304718018, |
| "logits/rejected": -1.6862401962280273, |
| "logps/chosen": -61.83301544189453, |
| "logps/rejected": -138.49703979492188, |
| "loss": 0.3442, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9937431812286377, |
| "rewards/margins": 2.239272356033325, |
| "rewards/rejected": -0.2455294132232666, |
| "step": 1130 |
| }, |
| { |
| "epoch": 0.41, |
| "grad_norm": 37.935713309108, |
| "learning_rate": 1.4483906348355185e-07, |
| "logits/chosen": -1.9341682195663452, |
| "logits/rejected": -1.8443982601165771, |
| "logps/chosen": -88.57161712646484, |
| "logps/rejected": -138.91732788085938, |
| "loss": 0.3578, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5548217296600342, |
| "rewards/margins": 1.6803414821624756, |
| "rewards/rejected": -0.1255197376012802, |
| "step": 1140 |
| }, |
| { |
| "epoch": 0.42, |
| "grad_norm": 41.30231239743997, |
| "learning_rate": 1.4466445721136494e-07, |
| "logits/chosen": -1.9577938318252563, |
| "logits/rejected": -1.6811765432357788, |
| "logps/chosen": -83.18925476074219, |
| "logps/rejected": -151.69259643554688, |
| "loss": 0.3935, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.5005887746810913, |
| "rewards/margins": 1.946763038635254, |
| "rewards/rejected": -0.4461742043495178, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.42, |
| "grad_norm": 47.452759957888006, |
| "learning_rate": 1.444870550411101e-07, |
| "logits/chosen": -2.1157777309417725, |
| "logits/rejected": -1.8774683475494385, |
| "logps/chosen": -72.8653793334961, |
| "logps/rejected": -136.59445190429688, |
| "loss": 0.3984, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.7430633306503296, |
| "rewards/margins": 1.85836660861969, |
| "rewards/rejected": -0.11530301719903946, |
| "step": 1160 |
| }, |
| { |
| "epoch": 0.42, |
| "grad_norm": 27.935593390748892, |
| "learning_rate": 1.443068640926072e-07, |
| "logits/chosen": -1.8945796489715576, |
| "logits/rejected": -2.1599068641662598, |
| "logps/chosen": -57.86497116088867, |
| "logps/rejected": -86.31175231933594, |
| "loss": 0.3544, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.740037202835083, |
| "rewards/margins": 1.9110946655273438, |
| "rewards/rejected": -0.171057790517807, |
| "step": 1170 |
| }, |
| { |
| "epoch": 0.43, |
| "grad_norm": 40.028733125905276, |
| "learning_rate": 1.4412389159760025e-07, |
| "logits/chosen": -2.241224527359009, |
| "logits/rejected": -1.6412010192871094, |
| "logps/chosen": -76.12211608886719, |
| "logps/rejected": -164.74478149414062, |
| "loss": 0.3165, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4326903820037842, |
| "rewards/margins": 2.377753496170044, |
| "rewards/rejected": -0.9450632333755493, |
| "step": 1180 |
| }, |
| { |
| "epoch": 0.43, |
| "grad_norm": 47.0201653078679, |
| "learning_rate": 1.439381448994673e-07, |
| "logits/chosen": -2.094806432723999, |
| "logits/rejected": -1.7619339227676392, |
| "logps/chosen": -80.14140319824219, |
| "logps/rejected": -172.28895568847656, |
| "loss": 0.3704, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.4097177982330322, |
| "rewards/margins": 2.00710129737854, |
| "rewards/rejected": -0.5973835587501526, |
| "step": 1190 |
| }, |
| { |
| "epoch": 0.44, |
| "grad_norm": 45.538770655344166, |
| "learning_rate": 1.4374963145292563e-07, |
| "logits/chosen": -2.0040104389190674, |
| "logits/rejected": -1.7236404418945312, |
| "logps/chosen": -78.05459594726562, |
| "logps/rejected": -118.63360595703125, |
| "loss": 0.3191, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.8405094146728516, |
| "rewards/margins": 1.999542236328125, |
| "rewards/rejected": -0.15903277695178986, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.44, |
| "eval_logits/chosen": -2.6045000553131104, |
| "eval_logits/rejected": -2.3742053508758545, |
| "eval_logps/chosen": -86.77649688720703, |
| "eval_logps/rejected": -132.74559020996094, |
| "eval_loss": 0.3634836673736572, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 1.3179771900177002, |
| "eval_rewards/margins": 2.0553901195526123, |
| "eval_rewards/rejected": -0.7374131083488464, |
| "eval_runtime": 71.874, |
| "eval_samples_per_second": 12.383, |
| "eval_steps_per_second": 0.195, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.44, |
| "grad_norm": 32.36206179132804, |
| "learning_rate": 1.4355835882373265e-07, |
| "logits/chosen": -1.9400993585586548, |
| "logits/rejected": -1.7014987468719482, |
| "logps/chosen": -102.5603256225586, |
| "logps/rejected": -158.36917114257812, |
| "loss": 0.3049, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.8652098774909973, |
| "rewards/margins": 1.5464653968811035, |
| "rewards/rejected": -0.6812552809715271, |
| "step": 1210 |
| }, |
| { |
| "epoch": 0.44, |
| "grad_norm": 42.82494337547035, |
| "learning_rate": 1.433643346883822e-07, |
| "logits/chosen": -2.0089356899261475, |
| "logits/rejected": -1.5017987489700317, |
| "logps/chosen": -91.1595687866211, |
| "logps/rejected": -135.00486755371094, |
| "loss": 0.3278, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4155638217926025, |
| "rewards/margins": 1.6915414333343506, |
| "rewards/rejected": -0.27597787976264954, |
| "step": 1220 |
| }, |
| { |
| "epoch": 0.45, |
| "grad_norm": 56.19237380245196, |
| "learning_rate": 1.4316756683379635e-07, |
| "logits/chosen": -1.520774483680725, |
| "logits/rejected": -1.4852197170257568, |
| "logps/chosen": -90.1426010131836, |
| "logps/rejected": -124.76286315917969, |
| "loss": 0.3839, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.9303890466690063, |
| "rewards/margins": 2.0999183654785156, |
| "rewards/rejected": -0.16952911019325256, |
| "step": 1230 |
| }, |
| { |
| "epoch": 0.45, |
| "grad_norm": 51.35397866660894, |
| "learning_rate": 1.4296806315701312e-07, |
| "logits/chosen": -2.2964136600494385, |
| "logits/rejected": -1.7400308847427368, |
| "logps/chosen": -50.99583053588867, |
| "logps/rejected": -114.6817626953125, |
| "loss": 0.3506, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.290008783340454, |
| "rewards/margins": 2.5690829753875732, |
| "rewards/rejected": -0.27907437086105347, |
| "step": 1240 |
| }, |
| { |
| "epoch": 0.45, |
| "grad_norm": 36.79699061086273, |
| "learning_rate": 1.427658316648694e-07, |
| "logits/chosen": -2.083348512649536, |
| "logits/rejected": -1.4933042526245117, |
| "logps/chosen": -73.86151885986328, |
| "logps/rejected": -149.9459991455078, |
| "loss": 0.3594, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.754785180091858, |
| "rewards/margins": 2.0723814964294434, |
| "rewards/rejected": -0.3175961375236511, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.46, |
| "grad_norm": 61.77575626305043, |
| "learning_rate": 1.4256088047367958e-07, |
| "logits/chosen": -2.3271241188049316, |
| "logits/rejected": -2.1036393642425537, |
| "logps/chosen": -58.489654541015625, |
| "logps/rejected": -135.50341796875, |
| "loss": 0.3875, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.7433359622955322, |
| "rewards/margins": 1.8952022790908813, |
| "rewards/rejected": -0.1518661081790924, |
| "step": 1260 |
| }, |
| { |
| "epoch": 0.46, |
| "grad_norm": 45.208373613406316, |
| "learning_rate": 1.423532178089099e-07, |
| "logits/chosen": -2.00785756111145, |
| "logits/rejected": -2.118194341659546, |
| "logps/chosen": -87.7576904296875, |
| "logps/rejected": -121.8628158569336, |
| "loss": 0.2697, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.2886258363723755, |
| "rewards/margins": 2.014240026473999, |
| "rewards/rejected": -0.7256141901016235, |
| "step": 1270 |
| }, |
| { |
| "epoch": 0.46, |
| "grad_norm": 35.53690989690113, |
| "learning_rate": 1.421428520048482e-07, |
| "logits/chosen": -1.6762386560440063, |
| "logits/rejected": -1.9535388946533203, |
| "logps/chosen": -91.85359191894531, |
| "logps/rejected": -145.9041290283203, |
| "loss": 0.3821, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.8587366342544556, |
| "rewards/margins": 1.5649387836456299, |
| "rewards/rejected": 0.2937980592250824, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.47, |
| "grad_norm": 33.054145363402554, |
| "learning_rate": 1.419297915042697e-07, |
| "logits/chosen": -2.3081910610198975, |
| "logits/rejected": -1.6348092555999756, |
| "logps/chosen": -73.78157806396484, |
| "logps/rejected": -153.98765563964844, |
| "loss": 0.3255, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.926365852355957, |
| "rewards/margins": 3.0679659843444824, |
| "rewards/rejected": -1.1416000127792358, |
| "step": 1290 |
| }, |
| { |
| "epoch": 0.47, |
| "grad_norm": 35.78921822753497, |
| "learning_rate": 1.4171404485809776e-07, |
| "logits/chosen": -1.6090662479400635, |
| "logits/rejected": -1.2726771831512451, |
| "logps/chosen": -80.95599365234375, |
| "logps/rejected": -132.03736877441406, |
| "loss": 0.3319, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.9143034219741821, |
| "rewards/margins": 1.9614051580429077, |
| "rewards/rejected": -0.04710172861814499, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.47, |
| "eval_logits/chosen": -2.6384994983673096, |
| "eval_logits/rejected": -2.401343822479248, |
| "eval_logps/chosen": -80.92295837402344, |
| "eval_logps/rejected": -127.90816497802734, |
| "eval_loss": 0.35564500093460083, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 1.9033304452896118, |
| "eval_rewards/margins": 2.1570017337799072, |
| "eval_rewards/rejected": -0.25367113947868347, |
| "eval_runtime": 72.0772, |
| "eval_samples_per_second": 12.348, |
| "eval_steps_per_second": 0.194, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 41.95871729273735, |
| "learning_rate": 1.4149562072506109e-07, |
| "logits/chosen": -1.6684595346450806, |
| "logits/rejected": -1.6990349292755127, |
| "logps/chosen": -106.24676513671875, |
| "logps/rejected": -139.57778930664062, |
| "loss": 0.3228, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.357491135597229, |
| "rewards/margins": 1.8578159809112549, |
| "rewards/rejected": -0.5003247261047363, |
| "step": 1310 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 29.34145064992561, |
| "learning_rate": 1.4127452787134597e-07, |
| "logits/chosen": -2.3935904502868652, |
| "logits/rejected": -1.9407179355621338, |
| "logps/chosen": -75.06128692626953, |
| "logps/rejected": -170.85089111328125, |
| "loss": 0.3365, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.364776372909546, |
| "rewards/margins": 1.6955333948135376, |
| "rewards/rejected": -0.33075690269470215, |
| "step": 1320 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 54.390170546748955, |
| "learning_rate": 1.4105077517024458e-07, |
| "logits/chosen": -2.1387906074523926, |
| "logits/rejected": -1.6588388681411743, |
| "logps/chosen": -64.23258972167969, |
| "logps/rejected": -124.39351654052734, |
| "loss": 0.3544, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.8940436840057373, |
| "rewards/margins": 2.4280219078063965, |
| "rewards/rejected": -0.5339787602424622, |
| "step": 1330 |
| }, |
| { |
| "epoch": 0.49, |
| "grad_norm": 41.328588380198035, |
| "learning_rate": 1.4082437160179884e-07, |
| "logits/chosen": -1.7820161581039429, |
| "logits/rejected": -1.5780375003814697, |
| "logps/chosen": -76.89183044433594, |
| "logps/rejected": -122.71630859375, |
| "loss": 0.3367, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.836066484451294, |
| "rewards/margins": 1.8271701335906982, |
| "rewards/rejected": 0.008896279148757458, |
| "step": 1340 |
| }, |
| { |
| "epoch": 0.49, |
| "grad_norm": 36.692279705097874, |
| "learning_rate": 1.4059532625243992e-07, |
| "logits/chosen": -1.921148657798767, |
| "logits/rejected": -1.6129744052886963, |
| "logps/chosen": -66.90629577636719, |
| "logps/rejected": -127.26859283447266, |
| "loss": 0.37, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4975523948669434, |
| "rewards/margins": 1.937352180480957, |
| "rewards/rejected": -0.43979987502098083, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.49, |
| "grad_norm": 30.243381015544752, |
| "learning_rate": 1.403636483146238e-07, |
| "logits/chosen": -2.2570159435272217, |
| "logits/rejected": -1.9846521615982056, |
| "logps/chosen": -79.24600982666016, |
| "logps/rejected": -115.38359069824219, |
| "loss": 0.3979, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5120487213134766, |
| "rewards/margins": 1.4373055696487427, |
| "rewards/rejected": 0.07474327087402344, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.5, |
| "grad_norm": 44.64162084878288, |
| "learning_rate": 1.40129347086462e-07, |
| "logits/chosen": -2.0078492164611816, |
| "logits/rejected": -1.9076550006866455, |
| "logps/chosen": -102.33550262451172, |
| "logps/rejected": -123.74501037597656, |
| "loss": 0.3483, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.0897409915924072, |
| "rewards/margins": 1.6122095584869385, |
| "rewards/rejected": -0.5224683880805969, |
| "step": 1370 |
| }, |
| { |
| "epoch": 0.5, |
| "grad_norm": 39.12577532028238, |
| "learning_rate": 1.3989243197134876e-07, |
| "logits/chosen": -2.399981737136841, |
| "logits/rejected": -1.599557876586914, |
| "logps/chosen": -82.2318115234375, |
| "logps/rejected": -135.64981079101562, |
| "loss": 0.3279, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5288795232772827, |
| "rewards/margins": 1.9530413150787354, |
| "rewards/rejected": -0.42416173219680786, |
| "step": 1380 |
| }, |
| { |
| "epoch": 0.5, |
| "grad_norm": 36.85635152662901, |
| "learning_rate": 1.396529124775834e-07, |
| "logits/chosen": -2.5128228664398193, |
| "logits/rejected": -2.076643705368042, |
| "logps/chosen": -105.89326477050781, |
| "logps/rejected": -146.3048553466797, |
| "loss": 0.3094, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3551980257034302, |
| "rewards/margins": 1.9240223169326782, |
| "rewards/rejected": -0.5688244700431824, |
| "step": 1390 |
| }, |
| { |
| "epoch": 0.51, |
| "grad_norm": 46.84425675735352, |
| "learning_rate": 1.394107982179888e-07, |
| "logits/chosen": -1.8977441787719727, |
| "logits/rejected": -2.266324996948242, |
| "logps/chosen": -78.88629913330078, |
| "logps/rejected": -171.7329559326172, |
| "loss": 0.3469, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.2071471214294434, |
| "rewards/margins": 1.8551435470581055, |
| "rewards/rejected": -0.6479963064193726, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.51, |
| "eval_logits/chosen": -2.633074998855591, |
| "eval_logits/rejected": -2.3969056606292725, |
| "eval_logps/chosen": -84.95893096923828, |
| "eval_logps/rejected": -132.71334838867188, |
| "eval_loss": 0.3462066352367401, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.4997329711914062, |
| "eval_rewards/margins": 2.2339236736297607, |
| "eval_rewards/rejected": -0.7341909408569336, |
| "eval_runtime": 71.9731, |
| "eval_samples_per_second": 12.366, |
| "eval_steps_per_second": 0.195, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.51, |
| "grad_norm": 28.479823112134607, |
| "learning_rate": 1.3916609890952566e-07, |
| "logits/chosen": -1.909106969833374, |
| "logits/rejected": -1.6904758214950562, |
| "logps/chosen": -82.46297454833984, |
| "logps/rejected": -124.34074401855469, |
| "loss": 0.319, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.6145960092544556, |
| "rewards/margins": 2.080094575881958, |
| "rewards/rejected": -0.4654986262321472, |
| "step": 1410 |
| }, |
| { |
| "epoch": 0.52, |
| "grad_norm": 39.355457607430694, |
| "learning_rate": 1.3891882437290242e-07, |
| "logits/chosen": -1.5750586986541748, |
| "logits/rejected": -1.6994893550872803, |
| "logps/chosen": -82.91178894042969, |
| "logps/rejected": -111.0701675415039, |
| "loss": 0.4042, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.5502707958221436, |
| "rewards/margins": 1.4318604469299316, |
| "rewards/rejected": 0.1184103712439537, |
| "step": 1420 |
| }, |
| { |
| "epoch": 0.52, |
| "grad_norm": 43.081802728687556, |
| "learning_rate": 1.386689845321812e-07, |
| "logits/chosen": -2.426974058151245, |
| "logits/rejected": -1.7977014780044556, |
| "logps/chosen": -79.90650939941406, |
| "logps/rejected": -139.31407165527344, |
| "loss": 0.3121, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8190940618515015, |
| "rewards/margins": 2.4520251750946045, |
| "rewards/rejected": -0.6329307556152344, |
| "step": 1430 |
| }, |
| { |
| "epoch": 0.52, |
| "grad_norm": 38.9495726615902, |
| "learning_rate": 1.384165894143794e-07, |
| "logits/chosen": -1.8546695709228516, |
| "logits/rejected": -1.7970068454742432, |
| "logps/chosen": -105.05964660644531, |
| "logps/rejected": -128.0200653076172, |
| "loss": 0.2894, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.3824260234832764, |
| "rewards/margins": 1.8351023197174072, |
| "rewards/rejected": -0.45267629623413086, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.53, |
| "grad_norm": 36.527576333516, |
| "learning_rate": 1.381616491490674e-07, |
| "logits/chosen": -2.3374876976013184, |
| "logits/rejected": -2.048158884048462, |
| "logps/chosen": -73.65496063232422, |
| "logps/rejected": -159.6743927001953, |
| "loss": 0.3077, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.549155592918396, |
| "rewards/margins": 2.265070915222168, |
| "rewards/rejected": -0.7159156799316406, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.53, |
| "grad_norm": 37.33453867519459, |
| "learning_rate": 1.3790417396796205e-07, |
| "logits/chosen": -1.6809660196304321, |
| "logits/rejected": -1.6903836727142334, |
| "logps/chosen": -92.21663665771484, |
| "logps/rejected": -123.54522705078125, |
| "loss": 0.3688, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5740840435028076, |
| "rewards/margins": 2.4830660820007324, |
| "rewards/rejected": -0.9089819192886353, |
| "step": 1460 |
| }, |
| { |
| "epoch": 0.53, |
| "grad_norm": 30.137594905849312, |
| "learning_rate": 1.376441742045158e-07, |
| "logits/chosen": -1.7785037755966187, |
| "logits/rejected": -1.5067006349563599, |
| "logps/chosen": -88.78409576416016, |
| "logps/rejected": -144.81686401367188, |
| "loss": 0.2812, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1298682689666748, |
| "rewards/margins": 2.1699564456939697, |
| "rewards/rejected": -1.0400878190994263, |
| "step": 1470 |
| }, |
| { |
| "epoch": 0.54, |
| "grad_norm": 45.12639127949661, |
| "learning_rate": 1.3738166029350223e-07, |
| "logits/chosen": -2.215520143508911, |
| "logits/rejected": -1.8298145532608032, |
| "logps/chosen": -78.92552185058594, |
| "logps/rejected": -147.58212280273438, |
| "loss": 0.361, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.8534799814224243, |
| "rewards/margins": 2.4630656242370605, |
| "rewards/rejected": -0.6095854640007019, |
| "step": 1480 |
| }, |
| { |
| "epoch": 0.54, |
| "grad_norm": 30.18520757137706, |
| "learning_rate": 1.3711664277059714e-07, |
| "logits/chosen": -1.3158643245697021, |
| "logits/rejected": -1.4519625902175903, |
| "logps/chosen": -84.15853118896484, |
| "logps/rejected": -140.84286499023438, |
| "loss": 0.3058, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.9669021368026733, |
| "rewards/margins": 2.4468729496002197, |
| "rewards/rejected": -0.4799710810184479, |
| "step": 1490 |
| }, |
| { |
| "epoch": 0.54, |
| "grad_norm": 39.46537040194847, |
| "learning_rate": 1.3684913227195577e-07, |
| "logits/chosen": -2.1749026775360107, |
| "logits/rejected": -1.9646120071411133, |
| "logps/chosen": -85.54203796386719, |
| "logps/rejected": -132.18484497070312, |
| "loss": 0.2976, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.073870062828064, |
| "rewards/margins": 1.4576705694198608, |
| "rewards/rejected": -0.3838004171848297, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.54, |
| "eval_logits/chosen": -2.6264164447784424, |
| "eval_logits/rejected": -2.3935818672180176, |
| "eval_logps/chosen": -86.55225372314453, |
| "eval_logps/rejected": -134.98130798339844, |
| "eval_loss": 0.3363237977027893, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.3404006958007812, |
| "eval_rewards/margins": 2.3013861179351807, |
| "eval_rewards/rejected": -0.960985541343689, |
| "eval_runtime": 71.9646, |
| "eval_samples_per_second": 12.367, |
| "eval_steps_per_second": 0.195, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.55, |
| "grad_norm": 46.836051305920115, |
| "learning_rate": 1.365791395337859e-07, |
| "logits/chosen": -1.5352973937988281, |
| "logits/rejected": -1.6385904550552368, |
| "logps/chosen": -79.53118896484375, |
| "logps/rejected": -113.24952697753906, |
| "loss": 0.3002, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.4074381589889526, |
| "rewards/margins": 2.091799259185791, |
| "rewards/rejected": -0.6843612790107727, |
| "step": 1510 |
| }, |
| { |
| "epoch": 0.55, |
| "grad_norm": 31.950241265489773, |
| "learning_rate": 1.3630667539191687e-07, |
| "logits/chosen": -1.526102066040039, |
| "logits/rejected": -1.5050265789031982, |
| "logps/chosen": -98.58650207519531, |
| "logps/rejected": -133.04754638671875, |
| "loss": 0.3186, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 1.1431519985198975, |
| "rewards/margins": 1.964644193649292, |
| "rewards/rejected": -0.8214923739433289, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.56, |
| "grad_norm": 40.788510789631786, |
| "learning_rate": 1.3603175078136497e-07, |
| "logits/chosen": -1.8123880624771118, |
| "logits/rejected": -1.3758784532546997, |
| "logps/chosen": -68.65928649902344, |
| "logps/rejected": -116.91877746582031, |
| "loss": 0.3174, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3894504308700562, |
| "rewards/margins": 2.661546468734741, |
| "rewards/rejected": -1.2720959186553955, |
| "step": 1530 |
| }, |
| { |
| "epoch": 0.56, |
| "grad_norm": 46.155078059787826, |
| "learning_rate": 1.3575437673589428e-07, |
| "logits/chosen": -1.7942472696304321, |
| "logits/rejected": -1.867285132408142, |
| "logps/chosen": -94.3653793334961, |
| "logps/rejected": -112.46723937988281, |
| "loss": 0.3213, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.7745468020439148, |
| "rewards/margins": 1.4602556228637695, |
| "rewards/rejected": -0.6857088804244995, |
| "step": 1540 |
| }, |
| { |
| "epoch": 0.56, |
| "grad_norm": 39.375049143103645, |
| "learning_rate": 1.3547456438757397e-07, |
| "logits/chosen": -2.056170701980591, |
| "logits/rejected": -1.2939419746398926, |
| "logps/chosen": -89.32717895507812, |
| "logps/rejected": -133.4949493408203, |
| "loss": 0.3142, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.4916505813598633, |
| "rewards/margins": 1.9974079132080078, |
| "rewards/rejected": -0.5057573914527893, |
| "step": 1550 |
| }, |
| { |
| "epoch": 0.57, |
| "grad_norm": 32.64828907806418, |
| "learning_rate": 1.3519232496633152e-07, |
| "logits/chosen": -2.2024412155151367, |
| "logits/rejected": -1.6808528900146484, |
| "logps/chosen": -64.60079193115234, |
| "logps/rejected": -127.1625747680664, |
| "loss": 0.3165, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7459484338760376, |
| "rewards/margins": 2.624204158782959, |
| "rewards/rejected": -0.8782557249069214, |
| "step": 1560 |
| }, |
| { |
| "epoch": 0.57, |
| "grad_norm": 37.74052323913691, |
| "learning_rate": 1.3490766979950211e-07, |
| "logits/chosen": -2.4449658393859863, |
| "logits/rejected": -2.1780996322631836, |
| "logps/chosen": -63.970916748046875, |
| "logps/rejected": -116.03575134277344, |
| "loss": 0.285, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5420334339141846, |
| "rewards/margins": 2.1397767066955566, |
| "rewards/rejected": -0.5977433919906616, |
| "step": 1570 |
| }, |
| { |
| "epoch": 0.57, |
| "grad_norm": 38.57331331754987, |
| "learning_rate": 1.3462061031137382e-07, |
| "logits/chosen": -1.8334707021713257, |
| "logits/rejected": -1.6402419805526733, |
| "logps/chosen": -81.07013702392578, |
| "logps/rejected": -196.51010131835938, |
| "loss": 0.3205, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.801913857460022, |
| "rewards/margins": 3.0623841285705566, |
| "rewards/rejected": -1.2604701519012451, |
| "step": 1580 |
| }, |
| { |
| "epoch": 0.58, |
| "grad_norm": 35.786587394552, |
| "learning_rate": 1.3433115802272929e-07, |
| "logits/chosen": -1.9113935232162476, |
| "logits/rejected": -1.5720059871673584, |
| "logps/chosen": -72.0071792602539, |
| "logps/rejected": -155.9192352294922, |
| "loss": 0.2904, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8154712915420532, |
| "rewards/margins": 2.6344165802001953, |
| "rewards/rejected": -0.8189449310302734, |
| "step": 1590 |
| }, |
| { |
| "epoch": 0.58, |
| "grad_norm": 38.20450363937764, |
| "learning_rate": 1.3403932455038328e-07, |
| "logits/chosen": -2.0501441955566406, |
| "logits/rejected": -1.8184171915054321, |
| "logps/chosen": -86.4954833984375, |
| "logps/rejected": -138.66053771972656, |
| "loss": 0.2839, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.814734697341919, |
| "rewards/margins": 2.979191303253174, |
| "rewards/rejected": -1.1644567251205444, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.58, |
| "eval_logits/chosen": -2.5862371921539307, |
| "eval_logits/rejected": -2.3639228343963623, |
| "eval_logps/chosen": -84.44693756103516, |
| "eval_logps/rejected": -134.8124237060547, |
| "eval_loss": 0.3325304090976715, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 1.5509331226348877, |
| "eval_rewards/margins": 2.4950315952301025, |
| "eval_rewards/rejected": -0.9440980553627014, |
| "eval_runtime": 71.9558, |
| "eval_samples_per_second": 12.369, |
| "eval_steps_per_second": 0.195, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.58, |
| "grad_norm": 35.827979868952966, |
| "learning_rate": 1.3374512160671644e-07, |
| "logits/chosen": -1.5930265188217163, |
| "logits/rejected": -1.6432759761810303, |
| "logps/chosen": -101.54774475097656, |
| "logps/rejected": -133.1485595703125, |
| "loss": 0.3038, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.5359328389167786, |
| "rewards/margins": 1.9368603229522705, |
| "rewards/rejected": -1.4009274244308472, |
| "step": 1610 |
| }, |
| { |
| "epoch": 0.59, |
| "grad_norm": 30.92177760123355, |
| "learning_rate": 1.3344856099920526e-07, |
| "logits/chosen": -1.9082187414169312, |
| "logits/rejected": -1.8227027654647827, |
| "logps/chosen": -100.11506652832031, |
| "logps/rejected": -131.5067596435547, |
| "loss": 0.3262, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.2241116762161255, |
| "rewards/margins": 1.7369029521942139, |
| "rewards/rejected": -0.5127913355827332, |
| "step": 1620 |
| }, |
| { |
| "epoch": 0.59, |
| "grad_norm": 37.15625275272501, |
| "learning_rate": 1.3314965462994826e-07, |
| "logits/chosen": -1.8317235708236694, |
| "logits/rejected": -1.373651146888733, |
| "logps/chosen": -88.5484848022461, |
| "logps/rejected": -131.82040405273438, |
| "loss": 0.2875, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.9509607553482056, |
| "rewards/margins": 2.071371555328369, |
| "rewards/rejected": -1.120410680770874, |
| "step": 1630 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 45.37302624095175, |
| "learning_rate": 1.3284841449518813e-07, |
| "logits/chosen": -1.686141014099121, |
| "logits/rejected": -1.2155983448028564, |
| "logps/chosen": -99.9353256225586, |
| "logps/rejected": -144.32980346679688, |
| "loss": 0.3014, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.9641411900520325, |
| "rewards/margins": 1.8706451654434204, |
| "rewards/rejected": -0.9065039753913879, |
| "step": 1640 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 55.85191593624814, |
| "learning_rate": 1.3254485268483055e-07, |
| "logits/chosen": -1.7119086980819702, |
| "logits/rejected": -1.548729419708252, |
| "logps/chosen": -92.55009460449219, |
| "logps/rejected": -211.9893341064453, |
| "loss": 0.3211, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1339528560638428, |
| "rewards/margins": 2.2448153495788574, |
| "rewards/rejected": -1.1108627319335938, |
| "step": 1650 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 36.20383771458562, |
| "learning_rate": 1.3223898138195864e-07, |
| "logits/chosen": -2.340451240539551, |
| "logits/rejected": -1.6691877841949463, |
| "logps/chosen": -57.80481719970703, |
| "logps/rejected": -119.7285385131836, |
| "loss": 0.2937, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.0053391456604004, |
| "rewards/margins": 2.7729382514953613, |
| "rewards/rejected": -0.7675992250442505, |
| "step": 1660 |
| }, |
| { |
| "epoch": 0.61, |
| "grad_norm": 33.03559928672513, |
| "learning_rate": 1.319308128623443e-07, |
| "logits/chosen": -1.8629367351531982, |
| "logits/rejected": -1.5715751647949219, |
| "logps/chosen": -94.56365203857422, |
| "logps/rejected": -130.30677795410156, |
| "loss": 0.2732, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.7679950594902039, |
| "rewards/margins": 2.2738566398620605, |
| "rewards/rejected": -1.5058612823486328, |
| "step": 1670 |
| }, |
| { |
| "epoch": 0.61, |
| "grad_norm": 39.357859474732834, |
| "learning_rate": 1.3162035949395548e-07, |
| "logits/chosen": -1.9703582525253296, |
| "logits/rejected": -1.6913745403289795, |
| "logps/chosen": -68.82579040527344, |
| "logps/rejected": -130.0358123779297, |
| "loss": 0.3041, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3682914972305298, |
| "rewards/margins": 2.242696762084961, |
| "rewards/rejected": -0.874405562877655, |
| "step": 1680 |
| }, |
| { |
| "epoch": 0.61, |
| "grad_norm": 44.19717913326496, |
| "learning_rate": 1.3130763373645956e-07, |
| "logits/chosen": -2.4028756618499756, |
| "logits/rejected": -1.5053786039352417, |
| "logps/chosen": -93.85604095458984, |
| "logps/rejected": -234.1709747314453, |
| "loss": 0.3071, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.169268250465393, |
| "rewards/margins": 3.234970808029175, |
| "rewards/rejected": -2.0657026767730713, |
| "step": 1690 |
| }, |
| { |
| "epoch": 0.62, |
| "grad_norm": 30.436503628615085, |
| "learning_rate": 1.309926481407237e-07, |
| "logits/chosen": -2.1171786785125732, |
| "logits/rejected": -1.6411173343658447, |
| "logps/chosen": -81.68711853027344, |
| "logps/rejected": -157.5670166015625, |
| "loss": 0.3095, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.531718373298645, |
| "rewards/margins": 2.767521858215332, |
| "rewards/rejected": -1.2358036041259766, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.62, |
| "eval_logits/chosen": -2.6024363040924072, |
| "eval_logits/rejected": -2.381307601928711, |
| "eval_logps/chosen": -88.0004653930664, |
| "eval_logps/rejected": -138.6060028076172, |
| "eval_loss": 0.3237309455871582, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.1955795288085938, |
| "eval_rewards/margins": 2.519036054611206, |
| "eval_rewards/rejected": -1.3234565258026123, |
| "eval_runtime": 71.9205, |
| "eval_samples_per_second": 12.375, |
| "eval_steps_per_second": 0.195, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.62, |
| "grad_norm": 36.27177663241429, |
| "learning_rate": 1.3067541534831074e-07, |
| "logits/chosen": -2.4922726154327393, |
| "logits/rejected": -1.9374639987945557, |
| "logps/chosen": -95.45484161376953, |
| "logps/rejected": -176.23080444335938, |
| "loss": 0.3039, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.721973180770874, |
| "rewards/margins": 2.6191511154174805, |
| "rewards/rejected": -1.897178053855896, |
| "step": 1710 |
| }, |
| { |
| "epoch": 0.62, |
| "grad_norm": 32.7184977007972, |
| "learning_rate": 1.303559480909721e-07, |
| "logits/chosen": -2.122936487197876, |
| "logits/rejected": -1.8037102222442627, |
| "logps/chosen": -88.16950988769531, |
| "logps/rejected": -144.52603149414062, |
| "loss": 0.2703, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.2837637662887573, |
| "rewards/margins": 2.390108585357666, |
| "rewards/rejected": -1.1063446998596191, |
| "step": 1720 |
| }, |
| { |
| "epoch": 0.63, |
| "grad_norm": 29.34834638089634, |
| "learning_rate": 1.3003425919013677e-07, |
| "logits/chosen": -1.9752366542816162, |
| "logits/rejected": -1.5035978555679321, |
| "logps/chosen": -87.86656188964844, |
| "logps/rejected": -164.7413330078125, |
| "loss": 0.2606, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.7725374698638916, |
| "rewards/margins": 3.28916597366333, |
| "rewards/rejected": -1.5166288614273071, |
| "step": 1730 |
| }, |
| { |
| "epoch": 0.63, |
| "grad_norm": 36.999465061787745, |
| "learning_rate": 1.2971036155639654e-07, |
| "logits/chosen": -1.8440446853637695, |
| "logits/rejected": -1.6909431219100952, |
| "logps/chosen": -92.90309143066406, |
| "logps/rejected": -143.26986694335938, |
| "loss": 0.3133, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0839173793792725, |
| "rewards/margins": 2.5392398834228516, |
| "rewards/rejected": -1.455322504043579, |
| "step": 1740 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 38.30391165242112, |
| "learning_rate": 1.293842681889882e-07, |
| "logits/chosen": -2.0229029655456543, |
| "logits/rejected": -1.4855544567108154, |
| "logps/chosen": -93.77799987792969, |
| "logps/rejected": -204.0419464111328, |
| "loss": 0.3124, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2000395059585571, |
| "rewards/margins": 2.4565627574920654, |
| "rewards/rejected": -1.2565232515335083, |
| "step": 1750 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 41.551439437509096, |
| "learning_rate": 1.290559921752715e-07, |
| "logits/chosen": -2.3583412170410156, |
| "logits/rejected": -2.0657455921173096, |
| "logps/chosen": -85.60011291503906, |
| "logps/rejected": -131.65814208984375, |
| "loss": 0.3016, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.3455216884613037, |
| "rewards/margins": 2.5204150676727295, |
| "rewards/rejected": -1.1748934984207153, |
| "step": 1760 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 30.177779462195463, |
| "learning_rate": 1.287255466902041e-07, |
| "logits/chosen": -1.8713395595550537, |
| "logits/rejected": -1.5737354755401611, |
| "logps/chosen": -73.5456314086914, |
| "logps/rejected": -161.8707275390625, |
| "loss": 0.2684, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3379333019256592, |
| "rewards/margins": 3.22587251663208, |
| "rewards/rejected": -1.887939214706421, |
| "step": 1770 |
| }, |
| { |
| "epoch": 0.65, |
| "grad_norm": 42.266893865591655, |
| "learning_rate": 1.2839294499581266e-07, |
| "logits/chosen": -1.8609529733657837, |
| "logits/rejected": -1.9660890102386475, |
| "logps/chosen": -90.25117492675781, |
| "logps/rejected": -139.0089111328125, |
| "loss": 0.2725, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.4452346563339233, |
| "rewards/margins": 2.1682021617889404, |
| "rewards/rejected": -0.7229675054550171, |
| "step": 1780 |
| }, |
| { |
| "epoch": 0.65, |
| "grad_norm": 45.909064669287716, |
| "learning_rate": 1.280582004406608e-07, |
| "logits/chosen": -2.2296082973480225, |
| "logits/rejected": -1.7719411849975586, |
| "logps/chosen": -91.33625793457031, |
| "logps/rejected": -132.76153564453125, |
| "loss": 0.3173, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.3035770654678345, |
| "rewards/margins": 2.369469165802002, |
| "rewards/rejected": -1.0658921003341675, |
| "step": 1790 |
| }, |
| { |
| "epoch": 0.65, |
| "grad_norm": 38.096350658013705, |
| "learning_rate": 1.2772132645931315e-07, |
| "logits/chosen": -1.8231449127197266, |
| "logits/rejected": -1.4982807636260986, |
| "logps/chosen": -77.42939758300781, |
| "logps/rejected": -142.36465454101562, |
| "loss": 0.2593, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.426496982574463, |
| "rewards/margins": 2.347156047821045, |
| "rewards/rejected": -0.9206587672233582, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.65, |
| "eval_logits/chosen": -2.613116502761841, |
| "eval_logits/rejected": -2.38425612449646, |
| "eval_logps/chosen": -87.31402587890625, |
| "eval_logps/rejected": -137.7794952392578, |
| "eval_loss": 0.3188331723213196, |
| "eval_rewards/accuracies": 0.8392857313156128, |
| "eval_rewards/chosen": 1.2642244100570679, |
| "eval_rewards/margins": 2.5050292015075684, |
| "eval_rewards/rejected": -1.240804672241211, |
| "eval_runtime": 71.9246, |
| "eval_samples_per_second": 12.374, |
| "eval_steps_per_second": 0.195, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.66, |
| "grad_norm": 39.50184113749164, |
| "learning_rate": 1.273823365717963e-07, |
| "logits/chosen": -1.7294566631317139, |
| "logits/rejected": -1.341367244720459, |
| "logps/chosen": -74.58464050292969, |
| "logps/rejected": -133.0430450439453, |
| "loss": 0.3023, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.7522623538970947, |
| "rewards/margins": 2.6015543937683105, |
| "rewards/rejected": -0.8492921590805054, |
| "step": 1810 |
| }, |
| { |
| "epoch": 0.66, |
| "grad_norm": 42.89748213165641, |
| "learning_rate": 1.270412443830562e-07, |
| "logits/chosen": -1.636803388595581, |
| "logits/rejected": -1.4411513805389404, |
| "logps/chosen": -90.65261840820312, |
| "logps/rejected": -148.75440979003906, |
| "loss": 0.2737, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.4128892421722412, |
| "rewards/margins": 2.5060081481933594, |
| "rewards/rejected": -1.0931187868118286, |
| "step": 1820 |
| }, |
| { |
| "epoch": 0.66, |
| "grad_norm": 49.9218877269305, |
| "learning_rate": 1.2669806358241194e-07, |
| "logits/chosen": -1.672900915145874, |
| "logits/rejected": -1.657602310180664, |
| "logps/chosen": -102.61698150634766, |
| "logps/rejected": -124.9526138305664, |
| "loss": 0.3258, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.2539775371551514, |
| "rewards/margins": 1.9778436422348022, |
| "rewards/rejected": -0.7238659858703613, |
| "step": 1830 |
| }, |
| { |
| "epoch": 0.67, |
| "grad_norm": 52.08013573210232, |
| "learning_rate": 1.2635280794300674e-07, |
| "logits/chosen": -1.9965959787368774, |
| "logits/rejected": -1.7998332977294922, |
| "logps/chosen": -89.28053283691406, |
| "logps/rejected": -126.0293960571289, |
| "loss": 0.2968, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1432512998580933, |
| "rewards/margins": 1.8423658609390259, |
| "rewards/rejected": -0.6991146802902222, |
| "step": 1840 |
| }, |
| { |
| "epoch": 0.67, |
| "grad_norm": 57.840988304922966, |
| "learning_rate": 1.260054913212547e-07, |
| "logits/chosen": -2.127389907836914, |
| "logits/rejected": -1.7502504587173462, |
| "logps/chosen": -71.455078125, |
| "logps/rejected": -115.15047454833984, |
| "loss": 0.3373, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.6597360372543335, |
| "rewards/margins": 2.4347267150878906, |
| "rewards/rejected": -0.7749907970428467, |
| "step": 1850 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 53.530462845694025, |
| "learning_rate": 1.2565612765628513e-07, |
| "logits/chosen": -1.9101688861846924, |
| "logits/rejected": -1.6512504816055298, |
| "logps/chosen": -74.90973663330078, |
| "logps/rejected": -138.97523498535156, |
| "loss": 0.3027, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.8204338550567627, |
| "rewards/margins": 2.453209161758423, |
| "rewards/rejected": -0.6327755451202393, |
| "step": 1860 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 37.85257138685388, |
| "learning_rate": 1.2530473096938278e-07, |
| "logits/chosen": -1.8988683223724365, |
| "logits/rejected": -1.6503827571868896, |
| "logps/chosen": -79.6770248413086, |
| "logps/rejected": -143.09071350097656, |
| "loss": 0.3155, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.498981237411499, |
| "rewards/margins": 2.404602527618408, |
| "rewards/rejected": -0.9056210517883301, |
| "step": 1870 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 50.64293606347689, |
| "learning_rate": 1.2495131536342536e-07, |
| "logits/chosen": -2.128816843032837, |
| "logits/rejected": -1.7718677520751953, |
| "logps/chosen": -86.67499542236328, |
| "logps/rejected": -179.9734344482422, |
| "loss": 0.2966, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5343300104141235, |
| "rewards/margins": 2.9166336059570312, |
| "rewards/rejected": -1.3823034763336182, |
| "step": 1880 |
| }, |
| { |
| "epoch": 0.69, |
| "grad_norm": 37.57626214701421, |
| "learning_rate": 1.245958950223174e-07, |
| "logits/chosen": -1.349810242652893, |
| "logits/rejected": -1.1940224170684814, |
| "logps/chosen": -87.54304504394531, |
| "logps/rejected": -124.6638412475586, |
| "loss": 0.3138, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.8737591505050659, |
| "rewards/margins": 1.981792688369751, |
| "rewards/rejected": -1.1080334186553955, |
| "step": 1890 |
| }, |
| { |
| "epoch": 0.69, |
| "grad_norm": 37.541514062279354, |
| "learning_rate": 1.24238484210421e-07, |
| "logits/chosen": -2.122223138809204, |
| "logits/rejected": -1.972744345664978, |
| "logps/chosen": -123.90694427490234, |
| "logps/rejected": -174.65133666992188, |
| "loss": 0.2394, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.4691895842552185, |
| "rewards/margins": 1.7866857051849365, |
| "rewards/rejected": -1.3174960613250732, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.69, |
| "eval_logits/chosen": -2.6142075061798096, |
| "eval_logits/rejected": -2.388277053833008, |
| "eval_logps/chosen": -86.47371673583984, |
| "eval_logps/rejected": -137.2868194580078, |
| "eval_loss": 0.3111128807067871, |
| "eval_rewards/accuracies": 0.8392857313156128, |
| "eval_rewards/chosen": 1.3482542037963867, |
| "eval_rewards/margins": 2.539792776107788, |
| "eval_rewards/rejected": -1.1915382146835327, |
| "eval_runtime": 71.9156, |
| "eval_samples_per_second": 12.376, |
| "eval_steps_per_second": 0.195, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.69, |
| "grad_norm": 48.134112820259595, |
| "learning_rate": 1.2387909727198345e-07, |
| "logits/chosen": -2.399280309677124, |
| "logits/rejected": -2.280487537384033, |
| "logps/chosen": -97.30809020996094, |
| "logps/rejected": -159.677734375, |
| "loss": 0.2845, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.0647090673446655, |
| "rewards/margins": 1.8406950235366821, |
| "rewards/rejected": -0.7759860754013062, |
| "step": 1910 |
| }, |
| { |
| "epoch": 0.7, |
| "grad_norm": 49.91211298121504, |
| "learning_rate": 1.2351774863056134e-07, |
| "logits/chosen": -2.3767571449279785, |
| "logits/rejected": -1.682460069656372, |
| "logps/chosen": -62.444793701171875, |
| "logps/rejected": -145.32229614257812, |
| "loss": 0.2989, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.9718739986419678, |
| "rewards/margins": 3.5472664833068848, |
| "rewards/rejected": -1.575392246246338, |
| "step": 1920 |
| }, |
| { |
| "epoch": 0.7, |
| "grad_norm": 29.38061476258355, |
| "learning_rate": 1.2315445278844197e-07, |
| "logits/chosen": -2.427912950515747, |
| "logits/rejected": -1.9351403713226318, |
| "logps/chosen": -73.01206970214844, |
| "logps/rejected": -209.5726776123047, |
| "loss": 0.2909, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.7785199880599976, |
| "rewards/margins": 2.738701581954956, |
| "rewards/rejected": -0.9601818323135376, |
| "step": 1930 |
| }, |
| { |
| "epoch": 0.7, |
| "grad_norm": 26.75959860175755, |
| "learning_rate": 1.227892243260611e-07, |
| "logits/chosen": -2.469193696975708, |
| "logits/rejected": -1.6766465902328491, |
| "logps/chosen": -56.01939010620117, |
| "logps/rejected": -119.8520736694336, |
| "loss": 0.3141, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.8703302145004272, |
| "rewards/margins": 2.945542335510254, |
| "rewards/rejected": -1.0752121210098267, |
| "step": 1940 |
| }, |
| { |
| "epoch": 0.71, |
| "grad_norm": 38.461850874562536, |
| "learning_rate": 1.224220779014178e-07, |
| "logits/chosen": -1.9026371240615845, |
| "logits/rejected": -1.4415004253387451, |
| "logps/chosen": -97.89253997802734, |
| "logps/rejected": -149.56936645507812, |
| "loss": 0.2902, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.6727228164672852, |
| "rewards/margins": 2.737921953201294, |
| "rewards/rejected": -1.0651990175247192, |
| "step": 1950 |
| }, |
| { |
| "epoch": 0.71, |
| "grad_norm": 31.657055351360636, |
| "learning_rate": 1.2205302824948634e-07, |
| "logits/chosen": -2.2927825450897217, |
| "logits/rejected": -1.8084007501602173, |
| "logps/chosen": -80.52983093261719, |
| "logps/rejected": -156.01187133789062, |
| "loss": 0.3306, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.4960755109786987, |
| "rewards/margins": 2.939119338989258, |
| "rewards/rejected": -1.4430434703826904, |
| "step": 1960 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 50.459905943712606, |
| "learning_rate": 1.2168209018162456e-07, |
| "logits/chosen": -2.1764161586761475, |
| "logits/rejected": -1.9086568355560303, |
| "logps/chosen": -97.29804992675781, |
| "logps/rejected": -158.05215454101562, |
| "loss": 0.2753, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2845994234085083, |
| "rewards/margins": 2.076698064804077, |
| "rewards/rejected": -0.7920988202095032, |
| "step": 1970 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 34.05499830411277, |
| "learning_rate": 1.2130927858497966e-07, |
| "logits/chosen": -1.833762526512146, |
| "logits/rejected": -1.881530523300171, |
| "logps/chosen": -108.65325927734375, |
| "logps/rejected": -144.40902709960938, |
| "loss": 0.3155, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.106397271156311, |
| "rewards/margins": 2.1092371940612793, |
| "rewards/rejected": -1.0028399229049683, |
| "step": 1980 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 18.84745830969425, |
| "learning_rate": 1.209346084218906e-07, |
| "logits/chosen": -1.7890899181365967, |
| "logits/rejected": -1.7402870655059814, |
| "logps/chosen": -86.61128234863281, |
| "logps/rejected": -116.84139251708984, |
| "loss": 0.3061, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.6292413473129272, |
| "rewards/margins": 2.007689952850342, |
| "rewards/rejected": -0.3784485161304474, |
| "step": 1990 |
| }, |
| { |
| "epoch": 0.73, |
| "grad_norm": 48.3959022523091, |
| "learning_rate": 1.2055809472928762e-07, |
| "logits/chosen": -1.7730016708374023, |
| "logits/rejected": -1.6386531591415405, |
| "logps/chosen": -79.35774230957031, |
| "logps/rejected": -126.07957458496094, |
| "loss": 0.3234, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5467567443847656, |
| "rewards/margins": 2.7957520484924316, |
| "rewards/rejected": -1.2489951848983765, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.73, |
| "eval_logits/chosen": -2.596472978591919, |
| "eval_logits/rejected": -2.3702468872070312, |
| "eval_logps/chosen": -86.58128356933594, |
| "eval_logps/rejected": -138.30990600585938, |
| "eval_loss": 0.3054315149784088, |
| "eval_rewards/accuracies": 0.8392857313156128, |
| "eval_rewards/chosen": 1.3374978303909302, |
| "eval_rewards/margins": 2.6313438415527344, |
| "eval_rewards/rejected": -1.2938461303710938, |
| "eval_runtime": 71.9938, |
| "eval_samples_per_second": 12.362, |
| "eval_steps_per_second": 0.194, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.73, |
| "grad_norm": 53.09826173444019, |
| "learning_rate": 1.2017975261808887e-07, |
| "logits/chosen": -1.9460725784301758, |
| "logits/rejected": -1.7900736331939697, |
| "logps/chosen": -103.51519775390625, |
| "logps/rejected": -141.5853271484375, |
| "loss": 0.3128, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3981924057006836, |
| "rewards/margins": 2.500655174255371, |
| "rewards/rejected": -1.1024627685546875, |
| "step": 2010 |
| }, |
| { |
| "epoch": 0.73, |
| "grad_norm": 39.783453163990046, |
| "learning_rate": 1.1979959727259367e-07, |
| "logits/chosen": -1.8045324087142944, |
| "logits/rejected": -1.6309808492660522, |
| "logps/chosen": -92.61442565917969, |
| "logps/rejected": -154.93099975585938, |
| "loss": 0.2678, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.2143778800964355, |
| "rewards/margins": 2.35176420211792, |
| "rewards/rejected": -1.137386441230774, |
| "step": 2020 |
| }, |
| { |
| "epoch": 0.74, |
| "grad_norm": 39.81682054465197, |
| "learning_rate": 1.1941764394987351e-07, |
| "logits/chosen": -1.738677740097046, |
| "logits/rejected": -1.5726416110992432, |
| "logps/chosen": -82.56022644042969, |
| "logps/rejected": -151.69943237304688, |
| "loss": 0.2551, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.240398645401001, |
| "rewards/margins": 2.4828732013702393, |
| "rewards/rejected": -1.2424746751785278, |
| "step": 2030 |
| }, |
| { |
| "epoch": 0.74, |
| "grad_norm": 37.01944950877088, |
| "learning_rate": 1.1903390797915929e-07, |
| "logits/chosen": -2.1611745357513428, |
| "logits/rejected": -1.4224140644073486, |
| "logps/chosen": -80.29561614990234, |
| "logps/rejected": -215.88174438476562, |
| "loss": 0.2887, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.6173490285873413, |
| "rewards/margins": 3.720074415206909, |
| "rewards/rejected": -2.1027255058288574, |
| "step": 2040 |
| }, |
| { |
| "epoch": 0.74, |
| "grad_norm": 40.82628805207018, |
| "learning_rate": 1.1864840476122647e-07, |
| "logits/chosen": -1.8892196416854858, |
| "logits/rejected": -1.9005800485610962, |
| "logps/chosen": -69.93404388427734, |
| "logps/rejected": -123.88677978515625, |
| "loss": 0.2882, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8273292779922485, |
| "rewards/margins": 3.182246685028076, |
| "rewards/rejected": -1.354917287826538, |
| "step": 2050 |
| }, |
| { |
| "epoch": 0.75, |
| "grad_norm": 25.39556496888905, |
| "learning_rate": 1.1826114976777678e-07, |
| "logits/chosen": -1.7499809265136719, |
| "logits/rejected": -1.4074931144714355, |
| "logps/chosen": -91.85431671142578, |
| "logps/rejected": -170.28402709960938, |
| "loss": 0.2753, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.5811419486999512, |
| "rewards/margins": 3.3795552253723145, |
| "rewards/rejected": -1.7984129190444946, |
| "step": 2060 |
| }, |
| { |
| "epoch": 0.75, |
| "grad_norm": 27.81061856916663, |
| "learning_rate": 1.1787215854081724e-07, |
| "logits/chosen": -2.056485652923584, |
| "logits/rejected": -1.6245391368865967, |
| "logps/chosen": -80.32780456542969, |
| "logps/rejected": -132.90272521972656, |
| "loss": 0.3135, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4287405014038086, |
| "rewards/margins": 2.6631085872650146, |
| "rewards/rejected": -1.2343682050704956, |
| "step": 2070 |
| }, |
| { |
| "epoch": 0.75, |
| "grad_norm": 30.567746366717504, |
| "learning_rate": 1.1748144669203663e-07, |
| "logits/chosen": -1.631363868713379, |
| "logits/rejected": -1.6502500772476196, |
| "logps/chosen": -96.3510971069336, |
| "logps/rejected": -155.26380920410156, |
| "loss": 0.2693, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1531099081039429, |
| "rewards/margins": 2.349030017852783, |
| "rewards/rejected": -1.1959199905395508, |
| "step": 2080 |
| }, |
| { |
| "epoch": 0.76, |
| "grad_norm": 40.52562795601512, |
| "learning_rate": 1.1708902990217868e-07, |
| "logits/chosen": -1.8120248317718506, |
| "logits/rejected": -1.6999822854995728, |
| "logps/chosen": -73.98417663574219, |
| "logps/rejected": -166.10572814941406, |
| "loss": 0.2698, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.971939742565155, |
| "rewards/margins": 3.178133487701416, |
| "rewards/rejected": -2.206193685531616, |
| "step": 2090 |
| }, |
| { |
| "epoch": 0.76, |
| "grad_norm": 42.93005039151693, |
| "learning_rate": 1.166949239204129e-07, |
| "logits/chosen": -2.2130112648010254, |
| "logits/rejected": -1.7474415302276611, |
| "logps/chosen": -89.34376525878906, |
| "logps/rejected": -188.7493133544922, |
| "loss": 0.2532, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.4905040264129639, |
| "rewards/margins": 3.4080872535705566, |
| "rewards/rejected": -1.9175831079483032, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.76, |
| "eval_logits/chosen": -2.5582356452941895, |
| "eval_logits/rejected": -2.3411316871643066, |
| "eval_logps/chosen": -86.91837310791016, |
| "eval_logps/rejected": -139.59432983398438, |
| "eval_loss": 0.30377882719039917, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 1.303788423538208, |
| "eval_rewards/margins": 2.72607684135437, |
| "eval_rewards/rejected": -1.4222885370254517, |
| "eval_runtime": 71.8831, |
| "eval_samples_per_second": 12.381, |
| "eval_steps_per_second": 0.195, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.77, |
| "grad_norm": 53.74378080351859, |
| "learning_rate": 1.1629914456370243e-07, |
| "logits/chosen": -1.6835792064666748, |
| "logits/rejected": -1.5631754398345947, |
| "logps/chosen": -79.44068908691406, |
| "logps/rejected": -109.39453125, |
| "loss": 0.3213, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.8239709734916687, |
| "rewards/margins": 1.709472417831421, |
| "rewards/rejected": -0.885501503944397, |
| "step": 2110 |
| }, |
| { |
| "epoch": 0.77, |
| "grad_norm": 45.76775103982164, |
| "learning_rate": 1.1590170771616929e-07, |
| "logits/chosen": -2.1422882080078125, |
| "logits/rejected": -2.01249361038208, |
| "logps/chosen": -70.21590423583984, |
| "logps/rejected": -106.15130615234375, |
| "loss": 0.2981, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.9929507970809937, |
| "rewards/margins": 1.7180888652801514, |
| "rewards/rejected": 0.27486199140548706, |
| "step": 2120 |
| }, |
| { |
| "epoch": 0.77, |
| "grad_norm": 34.522061772377945, |
| "learning_rate": 1.1550262932845691e-07, |
| "logits/chosen": -1.712194800376892, |
| "logits/rejected": -1.602046251296997, |
| "logps/chosen": -87.20169830322266, |
| "logps/rejected": -150.86685180664062, |
| "loss": 0.2747, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.7278430461883545, |
| "rewards/margins": 2.8564956188201904, |
| "rewards/rejected": -1.1286523342132568, |
| "step": 2130 |
| }, |
| { |
| "epoch": 0.78, |
| "grad_norm": 54.87111432142923, |
| "learning_rate": 1.1510192541708984e-07, |
| "logits/chosen": -1.5016837120056152, |
| "logits/rejected": -1.4684686660766602, |
| "logps/chosen": -72.81539916992188, |
| "logps/rejected": -142.9152374267578, |
| "loss": 0.3061, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.2450437545776367, |
| "rewards/margins": 3.1693520545959473, |
| "rewards/rejected": -0.9243084192276001, |
| "step": 2140 |
| }, |
| { |
| "epoch": 0.78, |
| "grad_norm": 50.33350490881025, |
| "learning_rate": 1.1469961206383114e-07, |
| "logits/chosen": -2.383873462677002, |
| "logits/rejected": -2.1281042098999023, |
| "logps/chosen": -93.19439697265625, |
| "logps/rejected": -159.3778076171875, |
| "loss": 0.2755, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0546815395355225, |
| "rewards/margins": 2.3399760723114014, |
| "rewards/rejected": -1.285294532775879, |
| "step": 2150 |
| }, |
| { |
| "epoch": 0.78, |
| "grad_norm": 29.94921093499752, |
| "learning_rate": 1.1429570541503681e-07, |
| "logits/chosen": -1.880446434020996, |
| "logits/rejected": -1.4601715803146362, |
| "logps/chosen": -97.71735382080078, |
| "logps/rejected": -151.2371368408203, |
| "loss": 0.2765, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.2635929584503174, |
| "rewards/margins": 3.5359420776367188, |
| "rewards/rejected": -2.2723495960235596, |
| "step": 2160 |
| }, |
| { |
| "epoch": 0.79, |
| "grad_norm": 38.26921815972567, |
| "learning_rate": 1.1389022168100782e-07, |
| "logits/chosen": -1.897870659828186, |
| "logits/rejected": -1.5864663124084473, |
| "logps/chosen": -91.51280212402344, |
| "logps/rejected": -171.35614013671875, |
| "loss": 0.2715, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.0029170513153076, |
| "rewards/margins": 2.7681167125701904, |
| "rewards/rejected": -1.7651996612548828, |
| "step": 2170 |
| }, |
| { |
| "epoch": 0.79, |
| "grad_norm": 25.55797410044776, |
| "learning_rate": 1.1348317713533955e-07, |
| "logits/chosen": -1.8892625570297241, |
| "logits/rejected": -1.941860556602478, |
| "logps/chosen": -112.3303451538086, |
| "logps/rejected": -152.99148559570312, |
| "loss": 0.2756, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.2206674814224243, |
| "rewards/margins": 2.4190514087677, |
| "rewards/rejected": -1.1983839273452759, |
| "step": 2180 |
| }, |
| { |
| "epoch": 0.79, |
| "grad_norm": 35.1411721341359, |
| "learning_rate": 1.1307458811426865e-07, |
| "logits/chosen": -2.288433313369751, |
| "logits/rejected": -1.941382646560669, |
| "logps/chosen": -117.0739517211914, |
| "logps/rejected": -211.00942993164062, |
| "loss": 0.2902, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.4456199109554291, |
| "rewards/margins": 3.088353395462036, |
| "rewards/rejected": -2.642733335494995, |
| "step": 2190 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 33.568721302560235, |
| "learning_rate": 1.1266447101601738e-07, |
| "logits/chosen": -2.234501361846924, |
| "logits/rejected": -1.5646653175354004, |
| "logps/chosen": -85.79940795898438, |
| "logps/rejected": -137.91712951660156, |
| "loss": 0.2862, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.6818853616714478, |
| "rewards/margins": 1.9311177730560303, |
| "rewards/rejected": -1.249232530593872, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.8, |
| "eval_logits/chosen": -2.611751079559326, |
| "eval_logits/rejected": -2.3868792057037354, |
| "eval_logps/chosen": -90.76590728759766, |
| "eval_logps/rejected": -142.14036560058594, |
| "eval_loss": 0.29903972148895264, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.9190365672111511, |
| "eval_rewards/margins": 2.59592866897583, |
| "eval_rewards/rejected": -1.6768922805786133, |
| "eval_runtime": 71.9273, |
| "eval_samples_per_second": 12.374, |
| "eval_steps_per_second": 0.195, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 40.79977336139254, |
| "learning_rate": 1.1225284230013554e-07, |
| "logits/chosen": -1.9493424892425537, |
| "logits/rejected": -1.7713226079940796, |
| "logps/chosen": -127.34124755859375, |
| "logps/rejected": -142.61746215820312, |
| "loss": 0.2739, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7090291976928711, |
| "rewards/margins": 2.365572452545166, |
| "rewards/rejected": -1.656543493270874, |
| "step": 2210 |
| }, |
| { |
| "epoch": 0.81, |
| "grad_norm": 42.792407711246895, |
| "learning_rate": 1.118397184868399e-07, |
| "logits/chosen": -1.76601243019104, |
| "logits/rejected": -1.4212825298309326, |
| "logps/chosen": -80.1509780883789, |
| "logps/rejected": -130.9561004638672, |
| "loss": 0.2638, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3400588035583496, |
| "rewards/margins": 2.613032817840576, |
| "rewards/rejected": -1.272973895072937, |
| "step": 2220 |
| }, |
| { |
| "epoch": 0.81, |
| "grad_norm": 32.9359485242729, |
| "learning_rate": 1.1142511615635106e-07, |
| "logits/chosen": -2.112626791000366, |
| "logits/rejected": -1.8563470840454102, |
| "logps/chosen": -96.79916381835938, |
| "logps/rejected": -137.69740295410156, |
| "loss": 0.271, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.3981750011444092, |
| "rewards/margins": 2.080976963043213, |
| "rewards/rejected": -0.6828019618988037, |
| "step": 2230 |
| }, |
| { |
| "epoch": 0.81, |
| "grad_norm": 31.24730889188293, |
| "learning_rate": 1.1100905194822817e-07, |
| "logits/chosen": -2.2185370922088623, |
| "logits/rejected": -1.9021923542022705, |
| "logps/chosen": -100.98394775390625, |
| "logps/rejected": -165.42442321777344, |
| "loss": 0.2959, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.7293106913566589, |
| "rewards/margins": 2.7436270713806152, |
| "rewards/rejected": -2.0143163204193115, |
| "step": 2240 |
| }, |
| { |
| "epoch": 0.82, |
| "grad_norm": 29.657646636917896, |
| "learning_rate": 1.1059154256070101e-07, |
| "logits/chosen": -2.116981029510498, |
| "logits/rejected": -1.6903644800186157, |
| "logps/chosen": -62.789337158203125, |
| "logps/rejected": -139.08433532714844, |
| "loss": 0.2711, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.8867229223251343, |
| "rewards/margins": 3.3395798206329346, |
| "rewards/rejected": -1.4528568983078003, |
| "step": 2250 |
| }, |
| { |
| "epoch": 0.82, |
| "grad_norm": 42.14718531203437, |
| "learning_rate": 1.101726047499999e-07, |
| "logits/chosen": -1.7964967489242554, |
| "logits/rejected": -1.3701756000518799, |
| "logps/chosen": -84.9123306274414, |
| "logps/rejected": -195.67616271972656, |
| "loss": 0.295, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.1344997882843018, |
| "rewards/margins": 2.1091301441192627, |
| "rewards/rejected": -0.9746300578117371, |
| "step": 2260 |
| }, |
| { |
| "epoch": 0.82, |
| "grad_norm": 44.98425619656123, |
| "learning_rate": 1.0975225532968324e-07, |
| "logits/chosen": -2.183009386062622, |
| "logits/rejected": -1.8169399499893188, |
| "logps/chosen": -71.13068389892578, |
| "logps/rejected": -153.55203247070312, |
| "loss": 0.2318, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.66214919090271, |
| "rewards/margins": 2.8192644119262695, |
| "rewards/rejected": -1.1571152210235596, |
| "step": 2270 |
| }, |
| { |
| "epoch": 0.83, |
| "grad_norm": 38.49859201574968, |
| "learning_rate": 1.0933051116996251e-07, |
| "logits/chosen": -2.2806153297424316, |
| "logits/rejected": -1.6658601760864258, |
| "logps/chosen": -87.88021850585938, |
| "logps/rejected": -157.28659057617188, |
| "loss": 0.2386, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.618966817855835, |
| "rewards/margins": 3.0835256576538086, |
| "rewards/rejected": -1.4645588397979736, |
| "step": 2280 |
| }, |
| { |
| "epoch": 0.83, |
| "grad_norm": 29.80803011600754, |
| "learning_rate": 1.089073891970255e-07, |
| "logits/chosen": -2.1777398586273193, |
| "logits/rejected": -1.9328248500823975, |
| "logps/chosen": -95.84873962402344, |
| "logps/rejected": -143.92617797851562, |
| "loss": 0.2572, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5284852981567383, |
| "rewards/margins": 3.2201919555664062, |
| "rewards/rejected": -1.6917064189910889, |
| "step": 2290 |
| }, |
| { |
| "epoch": 0.83, |
| "grad_norm": 38.40290299546086, |
| "learning_rate": 1.0848290639235677e-07, |
| "logits/chosen": -2.0058891773223877, |
| "logits/rejected": -1.5227512121200562, |
| "logps/chosen": -52.72210693359375, |
| "logps/rejected": -111.29896545410156, |
| "loss": 0.2972, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.021421432495117, |
| "rewards/margins": 2.641392230987549, |
| "rewards/rejected": -0.6199706196784973, |
| "step": 2300 |
| }, |
| { |
| "epoch": 0.83, |
| "eval_logits/chosen": -2.568045139312744, |
| "eval_logits/rejected": -2.3515257835388184, |
| "eval_logps/chosen": -86.87254333496094, |
| "eval_logps/rejected": -140.3467559814453, |
| "eval_loss": 0.29616376757621765, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 1.308371901512146, |
| "eval_rewards/margins": 2.8059024810791016, |
| "eval_rewards/rejected": -1.4975303411483765, |
| "eval_runtime": 71.9446, |
| "eval_samples_per_second": 12.371, |
| "eval_steps_per_second": 0.195, |
| "step": 2300 |
| }, |
| { |
| "epoch": 0.84, |
| "grad_norm": 32.78552611053422, |
| "learning_rate": 1.0805707979205626e-07, |
| "logits/chosen": -1.8444101810455322, |
| "logits/rejected": -1.6570831537246704, |
| "logps/chosen": -82.16907501220703, |
| "logps/rejected": -161.206787109375, |
| "loss": 0.2583, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.7756086587905884, |
| "rewards/margins": 3.3080837726593018, |
| "rewards/rejected": -1.532475233078003, |
| "step": 2310 |
| }, |
| { |
| "epoch": 0.84, |
| "grad_norm": 49.48891626378323, |
| "learning_rate": 1.0762992648615549e-07, |
| "logits/chosen": -2.2481982707977295, |
| "logits/rejected": -1.626535415649414, |
| "logps/chosen": -60.07440185546875, |
| "logps/rejected": -154.71829223632812, |
| "loss": 0.2713, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.572458028793335, |
| "rewards/margins": 2.8353686332702637, |
| "rewards/rejected": -1.2629106044769287, |
| "step": 2320 |
| }, |
| { |
| "epoch": 0.85, |
| "grad_norm": 32.324264844561206, |
| "learning_rate": 1.0720146361793166e-07, |
| "logits/chosen": -1.3140501976013184, |
| "logits/rejected": -1.3236348628997803, |
| "logps/chosen": -84.41221618652344, |
| "logps/rejected": -151.84896850585938, |
| "loss": 0.2769, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4236546754837036, |
| "rewards/margins": 3.031369686126709, |
| "rewards/rejected": -1.6077148914337158, |
| "step": 2330 |
| }, |
| { |
| "epoch": 0.85, |
| "grad_norm": 33.985147386077095, |
| "learning_rate": 1.0677170838321969e-07, |
| "logits/chosen": -2.1076178550720215, |
| "logits/rejected": -1.5049619674682617, |
| "logps/chosen": -63.42456817626953, |
| "logps/rejected": -135.26893615722656, |
| "loss": 0.3078, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8713020086288452, |
| "rewards/margins": 3.84709095954895, |
| "rewards/rejected": -1.9757890701293945, |
| "step": 2340 |
| }, |
| { |
| "epoch": 0.85, |
| "grad_norm": 28.49155730270651, |
| "learning_rate": 1.0634067802972204e-07, |
| "logits/chosen": -2.251544713973999, |
| "logits/rejected": -1.766570806503296, |
| "logps/chosen": -59.371925354003906, |
| "logps/rejected": -96.25486755371094, |
| "loss": 0.2679, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.6637605428695679, |
| "rewards/margins": 1.9050308465957642, |
| "rewards/rejected": -0.24127018451690674, |
| "step": 2350 |
| }, |
| { |
| "epoch": 0.86, |
| "grad_norm": 46.79774181330583, |
| "learning_rate": 1.0590838985631653e-07, |
| "logits/chosen": -1.972684621810913, |
| "logits/rejected": -1.8693819046020508, |
| "logps/chosen": -81.91486358642578, |
| "logps/rejected": -135.78457641601562, |
| "loss": 0.2746, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4232879877090454, |
| "rewards/margins": 3.040739059448242, |
| "rewards/rejected": -1.6174514293670654, |
| "step": 2360 |
| }, |
| { |
| "epoch": 0.86, |
| "grad_norm": 45.90541036194016, |
| "learning_rate": 1.0547486121236202e-07, |
| "logits/chosen": -2.168940305709839, |
| "logits/rejected": -1.6793582439422607, |
| "logps/chosen": -85.18965148925781, |
| "logps/rejected": -126.7214126586914, |
| "loss": 0.25, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3474589586257935, |
| "rewards/margins": 2.820422649383545, |
| "rewards/rejected": -1.472963571548462, |
| "step": 2370 |
| }, |
| { |
| "epoch": 0.86, |
| "grad_norm": 32.321114428237806, |
| "learning_rate": 1.0504010949700214e-07, |
| "logits/chosen": -2.089189052581787, |
| "logits/rejected": -1.6855659484863281, |
| "logps/chosen": -106.45992279052734, |
| "logps/rejected": -163.7842559814453, |
| "loss": 0.2809, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.0083647966384888, |
| "rewards/margins": 2.169463872909546, |
| "rewards/rejected": -1.1610991954803467, |
| "step": 2380 |
| }, |
| { |
| "epoch": 0.87, |
| "grad_norm": 29.03655328835733, |
| "learning_rate": 1.0460415215846706e-07, |
| "logits/chosen": -1.7703535556793213, |
| "logits/rejected": -1.6672289371490479, |
| "logps/chosen": -78.27106475830078, |
| "logps/rejected": -119.7970962524414, |
| "loss": 0.3083, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5671287775039673, |
| "rewards/margins": 2.6943907737731934, |
| "rewards/rejected": -1.1272616386413574, |
| "step": 2390 |
| }, |
| { |
| "epoch": 0.87, |
| "grad_norm": 40.17380893976502, |
| "learning_rate": 1.0416700669337309e-07, |
| "logits/chosen": -1.7051159143447876, |
| "logits/rejected": -1.4203988313674927, |
| "logps/chosen": -81.63056945800781, |
| "logps/rejected": -137.97152709960938, |
| "loss": 0.2819, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.8759418725967407, |
| "rewards/margins": 2.198500156402588, |
| "rewards/rejected": -1.3225582838058472, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.87, |
| "eval_logits/chosen": -2.5985686779022217, |
| "eval_logits/rejected": -2.3792624473571777, |
| "eval_logps/chosen": -89.82866668701172, |
| "eval_logps/rejected": -142.37832641601562, |
| "eval_loss": 0.29318711161613464, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 1.0127601623535156, |
| "eval_rewards/margins": 2.7134480476379395, |
| "eval_rewards/rejected": -1.7006880044937134, |
| "eval_runtime": 71.9548, |
| "eval_samples_per_second": 12.369, |
| "eval_steps_per_second": 0.195, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.87, |
| "grad_norm": 33.42712229281822, |
| "learning_rate": 1.0372869064602057e-07, |
| "logits/chosen": -1.7892534732818604, |
| "logits/rejected": -1.1852535009384155, |
| "logps/chosen": -83.11079406738281, |
| "logps/rejected": -178.16806030273438, |
| "loss": 0.2442, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.063844084739685, |
| "rewards/margins": 2.93255352973938, |
| "rewards/rejected": -1.8687092065811157, |
| "step": 2410 |
| }, |
| { |
| "epoch": 0.88, |
| "grad_norm": 35.05254862902216, |
| "learning_rate": 1.032892216076898e-07, |
| "logits/chosen": -1.8080724477767944, |
| "logits/rejected": -2.15615177154541, |
| "logps/chosen": -103.28878021240234, |
| "logps/rejected": -124.3226089477539, |
| "loss": 0.2772, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.9695222973823547, |
| "rewards/margins": 1.7081365585327148, |
| "rewards/rejected": -0.7386142611503601, |
| "step": 2420 |
| }, |
| { |
| "epoch": 0.88, |
| "grad_norm": 36.27734663918374, |
| "learning_rate": 1.0284861721593486e-07, |
| "logits/chosen": -1.5924618244171143, |
| "logits/rejected": -1.227879285812378, |
| "logps/chosen": -101.4988784790039, |
| "logps/rejected": -178.65904235839844, |
| "loss": 0.2531, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.11053466796875, |
| "rewards/margins": 3.443126678466797, |
| "rewards/rejected": -2.3325917720794678, |
| "step": 2430 |
| }, |
| { |
| "epoch": 0.89, |
| "grad_norm": 24.345274978234222, |
| "learning_rate": 1.0240689515387594e-07, |
| "logits/chosen": -1.6779060363769531, |
| "logits/rejected": -1.6044038534164429, |
| "logps/chosen": -79.8328857421875, |
| "logps/rejected": -146.0101776123047, |
| "loss": 0.2333, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1275861263275146, |
| "rewards/margins": 3.003176212310791, |
| "rewards/rejected": -1.8755900859832764, |
| "step": 2440 |
| }, |
| { |
| "epoch": 0.89, |
| "grad_norm": 38.18691013378142, |
| "learning_rate": 1.0196407314948948e-07, |
| "logits/chosen": -2.0358641147613525, |
| "logits/rejected": -1.629227638244629, |
| "logps/chosen": -68.20674896240234, |
| "logps/rejected": -162.77389526367188, |
| "loss": 0.2482, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.680604338645935, |
| "rewards/margins": 3.843975067138672, |
| "rewards/rejected": -2.1633706092834473, |
| "step": 2450 |
| }, |
| { |
| "epoch": 0.89, |
| "grad_norm": 48.20820417201082, |
| "learning_rate": 1.015201689748969e-07, |
| "logits/chosen": -1.8446376323699951, |
| "logits/rejected": -1.5708643198013306, |
| "logps/chosen": -71.98088836669922, |
| "logps/rejected": -119.35798645019531, |
| "loss": 0.2679, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3860020637512207, |
| "rewards/margins": 2.888756275177002, |
| "rewards/rejected": -1.5027542114257812, |
| "step": 2460 |
| }, |
| { |
| "epoch": 0.9, |
| "grad_norm": 54.24613191962403, |
| "learning_rate": 1.0107520044565107e-07, |
| "logits/chosen": -1.6053078174591064, |
| "logits/rejected": -1.7150065898895264, |
| "logps/chosen": -95.47930908203125, |
| "logps/rejected": -129.6278839111328, |
| "loss": 0.2346, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3713977336883545, |
| "rewards/margins": 1.8234504461288452, |
| "rewards/rejected": -0.45205289125442505, |
| "step": 2470 |
| }, |
| { |
| "epoch": 0.9, |
| "grad_norm": 33.14197724163633, |
| "learning_rate": 1.0062918542002149e-07, |
| "logits/chosen": -1.7330402135849, |
| "logits/rejected": -1.5148823261260986, |
| "logps/chosen": -90.85403442382812, |
| "logps/rejected": -120.53018951416016, |
| "loss": 0.291, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.1606512069702148, |
| "rewards/margins": 1.8089545965194702, |
| "rewards/rejected": -0.6483034491539001, |
| "step": 2480 |
| }, |
| { |
| "epoch": 0.9, |
| "grad_norm": 37.842170372492035, |
| "learning_rate": 1.0018214179827752e-07, |
| "logits/chosen": -1.585915207862854, |
| "logits/rejected": -1.2993358373641968, |
| "logps/chosen": -100.29901123046875, |
| "logps/rejected": -149.75497436523438, |
| "loss": 0.2526, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.6282235980033875, |
| "rewards/margins": 2.4122531414031982, |
| "rewards/rejected": -1.784029245376587, |
| "step": 2490 |
| }, |
| { |
| "epoch": 0.91, |
| "grad_norm": 47.99587426386816, |
| "learning_rate": 9.973408752196995e-08, |
| "logits/chosen": -1.995279312133789, |
| "logits/rejected": -1.883319616317749, |
| "logps/chosen": -85.00283813476562, |
| "logps/rejected": -133.04844665527344, |
| "loss": 0.2523, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3437588214874268, |
| "rewards/margins": 2.9578824043273926, |
| "rewards/rejected": -1.6141233444213867, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.91, |
| "eval_logits/chosen": -2.529489755630493, |
| "eval_logits/rejected": -2.3238425254821777, |
| "eval_logps/chosen": -87.53929901123047, |
| "eval_logps/rejected": -142.36253356933594, |
| "eval_loss": 0.2887308895587921, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 1.2416969537734985, |
| "eval_rewards/margins": 2.9408047199249268, |
| "eval_rewards/rejected": -1.6991075277328491, |
| "eval_runtime": 71.9193, |
| "eval_samples_per_second": 12.375, |
| "eval_steps_per_second": 0.195, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.91, |
| "grad_norm": 43.66156972073303, |
| "learning_rate": 9.9285040573211e-08, |
| "logits/chosen": -1.346799612045288, |
| "logits/rejected": -1.2726528644561768, |
| "logps/chosen": -84.28345489501953, |
| "logps/rejected": -163.90647888183594, |
| "loss": 0.2379, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.04119873046875, |
| "rewards/margins": 2.8771090507507324, |
| "rewards/rejected": -1.8359102010726929, |
| "step": 2510 |
| }, |
| { |
| "epoch": 0.91, |
| "grad_norm": 29.983525537713383, |
| "learning_rate": 9.883501897395255e-08, |
| "logits/chosen": -2.4633562564849854, |
| "logits/rejected": -1.8329979181289673, |
| "logps/chosen": -77.02892303466797, |
| "logps/rejected": -148.3466796875, |
| "loss": 0.309, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.9383471012115479, |
| "rewards/margins": 3.8611648082733154, |
| "rewards/rejected": -1.9228184223175049, |
| "step": 2520 |
| }, |
| { |
| "epoch": 0.92, |
| "grad_norm": 54.586263258672254, |
| "learning_rate": 9.838404078526287e-08, |
| "logits/chosen": -1.8038403987884521, |
| "logits/rejected": -1.38016676902771, |
| "logps/chosen": -81.94984436035156, |
| "logps/rejected": -152.57546997070312, |
| "loss": 0.2406, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.466467022895813, |
| "rewards/margins": 3.583028793334961, |
| "rewards/rejected": -2.1165618896484375, |
| "step": 2530 |
| }, |
| { |
| "epoch": 0.92, |
| "grad_norm": 34.64469687487794, |
| "learning_rate": 9.79321241066019e-08, |
| "logits/chosen": -1.7441418170928955, |
| "logits/rejected": -1.8793613910675049, |
| "logps/chosen": -80.00519561767578, |
| "logps/rejected": -135.61685180664062, |
| "loss": 0.2527, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3446810245513916, |
| "rewards/margins": 2.6392345428466797, |
| "rewards/rejected": -1.2945533990859985, |
| "step": 2540 |
| }, |
| { |
| "epoch": 0.93, |
| "grad_norm": 52.236703640195636, |
| "learning_rate": 9.747928707509452e-08, |
| "logits/chosen": -2.3760972023010254, |
| "logits/rejected": -1.7141082286834717, |
| "logps/chosen": -70.12747955322266, |
| "logps/rejected": -142.975341796875, |
| "loss": 0.2482, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.4635976552963257, |
| "rewards/margins": 3.061598300933838, |
| "rewards/rejected": -1.5980006456375122, |
| "step": 2550 |
| }, |
| { |
| "epoch": 0.93, |
| "grad_norm": 54.072926051989626, |
| "learning_rate": 9.702554786480307e-08, |
| "logits/chosen": -2.1945765018463135, |
| "logits/rejected": -1.9323997497558594, |
| "logps/chosen": -109.21195983886719, |
| "logps/rejected": -147.5604705810547, |
| "loss": 0.2711, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.7500441670417786, |
| "rewards/margins": 1.275587558746338, |
| "rewards/rejected": -0.5255435109138489, |
| "step": 2560 |
| }, |
| { |
| "epoch": 0.93, |
| "grad_norm": 28.839686251981885, |
| "learning_rate": 9.657092468599762e-08, |
| "logits/chosen": -1.9283406734466553, |
| "logits/rejected": -1.9637447595596313, |
| "logps/chosen": -73.35202026367188, |
| "logps/rejected": -218.9588165283203, |
| "loss": 0.2294, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.6155729293823242, |
| "rewards/margins": 3.6671338081359863, |
| "rewards/rejected": -2.051560640335083, |
| "step": 2570 |
| }, |
| { |
| "epoch": 0.94, |
| "grad_norm": 23.167116682862034, |
| "learning_rate": 9.611543578442521e-08, |
| "logits/chosen": -2.004908800125122, |
| "logits/rejected": -1.8643825054168701, |
| "logps/chosen": -73.96257019042969, |
| "logps/rejected": -120.63322448730469, |
| "loss": 0.2663, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.1677210330963135, |
| "rewards/margins": 1.7092103958129883, |
| "rewards/rejected": -0.5414894819259644, |
| "step": 2580 |
| }, |
| { |
| "epoch": 0.94, |
| "grad_norm": 37.61703553720983, |
| "learning_rate": 9.565909944057774e-08, |
| "logits/chosen": -1.8392232656478882, |
| "logits/rejected": -1.622847318649292, |
| "logps/chosen": -98.19245910644531, |
| "logps/rejected": -152.8502197265625, |
| "loss": 0.2644, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.2672936916351318, |
| "rewards/margins": 2.7811713218688965, |
| "rewards/rejected": -1.513877511024475, |
| "step": 2590 |
| }, |
| { |
| "epoch": 0.94, |
| "grad_norm": 27.461231350463876, |
| "learning_rate": 9.520193396895798e-08, |
| "logits/chosen": -1.9339004755020142, |
| "logits/rejected": -1.5946998596191406, |
| "logps/chosen": -107.6896743774414, |
| "logps/rejected": -166.02828979492188, |
| "loss": 0.2534, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.9528090357780457, |
| "rewards/margins": 3.083143949508667, |
| "rewards/rejected": -2.1303353309631348, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.94, |
| "eval_logits/chosen": -2.55859375, |
| "eval_logits/rejected": -2.3451855182647705, |
| "eval_logps/chosen": -89.4637680053711, |
| "eval_logps/rejected": -143.6739959716797, |
| "eval_loss": 0.2875581979751587, |
| "eval_rewards/accuracies": 0.8214285969734192, |
| "eval_rewards/chosen": 1.0492494106292725, |
| "eval_rewards/margins": 2.8795037269592285, |
| "eval_rewards/rejected": -1.8302545547485352, |
| "eval_runtime": 71.9243, |
| "eval_samples_per_second": 12.374, |
| "eval_steps_per_second": 0.195, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.95, |
| "grad_norm": 49.8667964660086, |
| "learning_rate": 9.474395771734493e-08, |
| "logits/chosen": -1.8184125423431396, |
| "logits/rejected": -1.5943410396575928, |
| "logps/chosen": -81.92977905273438, |
| "logps/rejected": -143.59848022460938, |
| "loss": 0.284, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8414875268936157, |
| "rewards/margins": 3.2987303733825684, |
| "rewards/rejected": -2.457242727279663, |
| "step": 2610 |
| }, |
| { |
| "epoch": 0.95, |
| "grad_norm": 30.852377454296445, |
| "learning_rate": 9.428518906605715e-08, |
| "logits/chosen": -1.6976007223129272, |
| "logits/rejected": -2.0197830200195312, |
| "logps/chosen": -77.92415618896484, |
| "logps/rejected": -120.9302749633789, |
| "loss": 0.2503, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.2503085136413574, |
| "rewards/margins": 2.8966543674468994, |
| "rewards/rejected": -1.646345853805542, |
| "step": 2620 |
| }, |
| { |
| "epoch": 0.95, |
| "grad_norm": 44.68222855190559, |
| "learning_rate": 9.382564642721517e-08, |
| "logits/chosen": -2.2393205165863037, |
| "logits/rejected": -1.8123022317886353, |
| "logps/chosen": -104.79426574707031, |
| "logps/rejected": -157.04183959960938, |
| "loss": 0.2891, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.9819310307502747, |
| "rewards/margins": 2.653965473175049, |
| "rewards/rejected": -1.6720342636108398, |
| "step": 2630 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 49.38339013216613, |
| "learning_rate": 9.336534824400267e-08, |
| "logits/chosen": -1.8587281703948975, |
| "logits/rejected": -1.5764485597610474, |
| "logps/chosen": -72.25798797607422, |
| "logps/rejected": -129.2059783935547, |
| "loss": 0.2643, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.856530785560608, |
| "rewards/margins": 3.2116611003875732, |
| "rewards/rejected": -1.3551304340362549, |
| "step": 2640 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 38.226700349391514, |
| "learning_rate": 9.290431298992605e-08, |
| "logits/chosen": -2.0723516941070557, |
| "logits/rejected": -1.8886346817016602, |
| "logps/chosen": -73.31155395507812, |
| "logps/rejected": -133.17425537109375, |
| "loss": 0.2182, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.8224437236785889, |
| "rewards/margins": 2.681816577911377, |
| "rewards/rejected": -0.8593727946281433, |
| "step": 2650 |
| }, |
| { |
| "epoch": 0.97, |
| "grad_norm": 27.790126692612404, |
| "learning_rate": 9.244255916807328e-08, |
| "logits/chosen": -1.9878448247909546, |
| "logits/rejected": -1.7479631900787354, |
| "logps/chosen": -75.09999084472656, |
| "logps/rejected": -106.42635345458984, |
| "loss": 0.2358, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3537728786468506, |
| "rewards/margins": 2.176225423812866, |
| "rewards/rejected": -0.8224524259567261, |
| "step": 2660 |
| }, |
| { |
| "epoch": 0.97, |
| "grad_norm": 45.32065506760666, |
| "learning_rate": 9.19801053103711e-08, |
| "logits/chosen": -2.0774807929992676, |
| "logits/rejected": -1.564305305480957, |
| "logps/chosen": -108.69667053222656, |
| "logps/rejected": -161.80776977539062, |
| "loss": 0.2735, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 1.1085962057113647, |
| "rewards/margins": 2.628162145614624, |
| "rewards/rejected": -1.5195659399032593, |
| "step": 2670 |
| }, |
| { |
| "epoch": 0.97, |
| "grad_norm": 42.93481884745555, |
| "learning_rate": 9.151696997684133e-08, |
| "logits/chosen": -1.8935344219207764, |
| "logits/rejected": -1.3089290857315063, |
| "logps/chosen": -82.37386322021484, |
| "logps/rejected": -136.06396484375, |
| "loss": 0.3265, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3590657711029053, |
| "rewards/margins": 2.706857681274414, |
| "rewards/rejected": -1.3477914333343506, |
| "step": 2680 |
| }, |
| { |
| "epoch": 0.98, |
| "grad_norm": 35.54137712720915, |
| "learning_rate": 9.105317175485603e-08, |
| "logits/chosen": -1.7412023544311523, |
| "logits/rejected": -1.2009631395339966, |
| "logps/chosen": -100.62733459472656, |
| "logps/rejected": -161.97164916992188, |
| "loss": 0.2636, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.8465662002563477, |
| "rewards/margins": 2.8156447410583496, |
| "rewards/rejected": -1.9690784215927124, |
| "step": 2690 |
| }, |
| { |
| "epoch": 0.98, |
| "grad_norm": 28.86922135565408, |
| "learning_rate": 9.058872925839145e-08, |
| "logits/chosen": -1.6413863897323608, |
| "logits/rejected": -1.4907596111297607, |
| "logps/chosen": -95.3847427368164, |
| "logps/rejected": -137.19778442382812, |
| "loss": 0.2065, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.9992202520370483, |
| "rewards/margins": 2.450819969177246, |
| "rewards/rejected": -1.4515998363494873, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.98, |
| "eval_logits/chosen": -2.584843158721924, |
| "eval_logits/rejected": -2.3644392490386963, |
| "eval_logps/chosen": -91.23661804199219, |
| "eval_logps/rejected": -144.7612762451172, |
| "eval_loss": 0.28060293197631836, |
| "eval_rewards/accuracies": 0.8571428656578064, |
| "eval_rewards/chosen": 0.8719648718833923, |
| "eval_rewards/margins": 2.8109474182128906, |
| "eval_rewards/rejected": -1.9389822483062744, |
| "eval_runtime": 72.0935, |
| "eval_samples_per_second": 12.345, |
| "eval_steps_per_second": 0.194, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.98, |
| "grad_norm": 33.738621565238745, |
| "learning_rate": 9.012366112728104e-08, |
| "logits/chosen": -2.308584213256836, |
| "logits/rejected": -1.8555558919906616, |
| "logps/chosen": -87.93582153320312, |
| "logps/rejected": -192.4984130859375, |
| "loss": 0.2612, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.67702716588974, |
| "rewards/margins": 2.732879161834717, |
| "rewards/rejected": -2.055852174758911, |
| "step": 2710 |
| }, |
| { |
| "epoch": 0.99, |
| "grad_norm": 27.441037937773388, |
| "learning_rate": 8.96579860264673e-08, |
| "logits/chosen": -1.98538339138031, |
| "logits/rejected": -1.8649771213531494, |
| "logps/chosen": -95.34701538085938, |
| "logps/rejected": -144.11663818359375, |
| "loss": 0.2566, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.9213436841964722, |
| "rewards/margins": 2.6342453956604004, |
| "rewards/rejected": -1.7129017114639282, |
| "step": 2720 |
| }, |
| { |
| "epoch": 0.99, |
| "grad_norm": 30.78636513438412, |
| "learning_rate": 8.919172264525274e-08, |
| "logits/chosen": -1.7649853229522705, |
| "logits/rejected": -1.6394439935684204, |
| "logps/chosen": -93.97325134277344, |
| "logps/rejected": -128.0108642578125, |
| "loss": 0.2684, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.8014779090881348, |
| "rewards/margins": 2.1937363147735596, |
| "rewards/rejected": -1.3922584056854248, |
| "step": 2730 |
| }, |
| { |
| "epoch": 0.99, |
| "grad_norm": 31.448648707859167, |
| "learning_rate": 8.872488969654978e-08, |
| "logits/chosen": -1.7622663974761963, |
| "logits/rejected": -1.6264762878417969, |
| "logps/chosen": -67.11592102050781, |
| "logps/rejected": -124.63407897949219, |
| "loss": 0.2686, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3815752267837524, |
| "rewards/margins": 2.803210735321045, |
| "rewards/rejected": -1.4216358661651611, |
| "step": 2740 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 39.421832880751474, |
| "learning_rate": 8.825750591612973e-08, |
| "logits/chosen": -2.0863993167877197, |
| "logits/rejected": -2.0298314094543457, |
| "logps/chosen": -88.62030029296875, |
| "logps/rejected": -142.9340057373047, |
| "loss": 0.261, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.233635663986206, |
| "rewards/margins": 2.6705703735351562, |
| "rewards/rejected": -1.4369348287582397, |
| "step": 2750 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 24.481557561032986, |
| "learning_rate": 8.778959006187086e-08, |
| "logits/chosen": -2.1946396827697754, |
| "logits/rejected": -2.0280699729919434, |
| "logps/chosen": -109.33675384521484, |
| "logps/rejected": -159.2801055908203, |
| "loss": 0.1921, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.6714259386062622, |
| "rewards/margins": 1.6729438304901123, |
| "rewards/rejected": -1.0015180110931396, |
| "step": 2760 |
| }, |
| { |
| "epoch": 1.01, |
| "grad_norm": 33.3000860606884, |
| "learning_rate": 8.732116091300562e-08, |
| "logits/chosen": -2.672719955444336, |
| "logits/rejected": -2.3178205490112305, |
| "logps/chosen": -108.69718170166016, |
| "logps/rejected": -154.65037536621094, |
| "loss": 0.1737, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7940966486930847, |
| "rewards/margins": 2.4711570739746094, |
| "rewards/rejected": -1.6770607233047485, |
| "step": 2770 |
| }, |
| { |
| "epoch": 1.01, |
| "grad_norm": 33.433009311142335, |
| "learning_rate": 8.685223726936686e-08, |
| "logits/chosen": -1.885528802871704, |
| "logits/rejected": -1.5169920921325684, |
| "logps/chosen": -77.01846313476562, |
| "logps/rejected": -153.978759765625, |
| "loss": 0.171, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.206106185913086, |
| "rewards/margins": 3.5677733421325684, |
| "rewards/rejected": -2.3616671562194824, |
| "step": 2780 |
| }, |
| { |
| "epoch": 1.01, |
| "grad_norm": 36.29017775178196, |
| "learning_rate": 8.638283795063337e-08, |
| "logits/chosen": -1.7784044742584229, |
| "logits/rejected": -1.95382559299469, |
| "logps/chosen": -102.05876922607422, |
| "logps/rejected": -136.20278930664062, |
| "loss": 0.1558, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0153976678848267, |
| "rewards/margins": 2.9986801147460938, |
| "rewards/rejected": -1.9832820892333984, |
| "step": 2790 |
| }, |
| { |
| "epoch": 1.02, |
| "grad_norm": 29.62169456800005, |
| "learning_rate": 8.591298179557462e-08, |
| "logits/chosen": -1.6251239776611328, |
| "logits/rejected": -1.2165063619613647, |
| "logps/chosen": -86.18495178222656, |
| "logps/rejected": -161.23684692382812, |
| "loss": 0.1669, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3276035785675049, |
| "rewards/margins": 4.416350364685059, |
| "rewards/rejected": -3.0887465476989746, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.02, |
| "eval_logits/chosen": -2.437558650970459, |
| "eval_logits/rejected": -2.259409189224243, |
| "eval_logps/chosen": -88.76131439208984, |
| "eval_logps/rejected": -145.60667419433594, |
| "eval_loss": 0.2792236804962158, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 1.1194943189620972, |
| "eval_rewards/margins": 3.143015146255493, |
| "eval_rewards/rejected": -2.0235207080841064, |
| "eval_runtime": 71.9382, |
| "eval_samples_per_second": 12.372, |
| "eval_steps_per_second": 0.195, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.02, |
| "grad_norm": 25.67903585470204, |
| "learning_rate": 8.544268766129463e-08, |
| "logits/chosen": -1.636718988418579, |
| "logits/rejected": -1.3525017499923706, |
| "logps/chosen": -91.5226821899414, |
| "logps/rejected": -152.884033203125, |
| "loss": 0.1626, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0812289714813232, |
| "rewards/margins": 3.153066873550415, |
| "rewards/rejected": -2.0718374252319336, |
| "step": 2810 |
| }, |
| { |
| "epoch": 1.02, |
| "grad_norm": 23.17233872525318, |
| "learning_rate": 8.497197442247518e-08, |
| "logits/chosen": -2.255305051803589, |
| "logits/rejected": -1.3946579694747925, |
| "logps/chosen": -73.07662963867188, |
| "logps/rejected": -160.0610809326172, |
| "loss": 0.1865, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7297379970550537, |
| "rewards/margins": 3.913317918777466, |
| "rewards/rejected": -2.183579683303833, |
| "step": 2820 |
| }, |
| { |
| "epoch": 1.03, |
| "grad_norm": 38.41529064635961, |
| "learning_rate": 8.45008609706183e-08, |
| "logits/chosen": -1.6799609661102295, |
| "logits/rejected": -1.7947546243667603, |
| "logps/chosen": -98.69215393066406, |
| "logps/rejected": -176.2200469970703, |
| "loss": 0.1551, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3261483907699585, |
| "rewards/margins": 2.8912904262542725, |
| "rewards/rejected": -1.565142273902893, |
| "step": 2830 |
| }, |
| { |
| "epoch": 1.03, |
| "grad_norm": 33.65938875922252, |
| "learning_rate": 8.402936621328809e-08, |
| "logits/chosen": -1.584078073501587, |
| "logits/rejected": -1.5029069185256958, |
| "logps/chosen": -78.0562515258789, |
| "logps/rejected": -129.6306915283203, |
| "loss": 0.1773, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 1.175910472869873, |
| "rewards/margins": 3.317577838897705, |
| "rewards/rejected": -2.141667366027832, |
| "step": 2840 |
| }, |
| { |
| "epoch": 1.03, |
| "grad_norm": 28.882074958874135, |
| "learning_rate": 8.355750907335185e-08, |
| "logits/chosen": -1.3962595462799072, |
| "logits/rejected": -1.2154072523117065, |
| "logps/chosen": -97.38468933105469, |
| "logps/rejected": -141.413330078125, |
| "loss": 0.1708, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9436796307563782, |
| "rewards/margins": 2.901923418045044, |
| "rewards/rejected": -1.9582436084747314, |
| "step": 2850 |
| }, |
| { |
| "epoch": 1.04, |
| "grad_norm": 48.57490152838243, |
| "learning_rate": 8.308530848822072e-08, |
| "logits/chosen": -1.322256088256836, |
| "logits/rejected": -1.3967430591583252, |
| "logps/chosen": -69.242919921875, |
| "logps/rejected": -106.2719955444336, |
| "loss": 0.167, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.6083996295928955, |
| "rewards/margins": 3.0584025382995605, |
| "rewards/rejected": -1.4500024318695068, |
| "step": 2860 |
| }, |
| { |
| "epoch": 1.04, |
| "grad_norm": 33.604690884057646, |
| "learning_rate": 8.261278340908956e-08, |
| "logits/chosen": -2.133439302444458, |
| "logits/rejected": -1.5575885772705078, |
| "logps/chosen": -82.22913360595703, |
| "logps/rejected": -155.5948028564453, |
| "loss": 0.1388, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.6941452026367188, |
| "rewards/margins": 3.734036922454834, |
| "rewards/rejected": -2.0398921966552734, |
| "step": 2870 |
| }, |
| { |
| "epoch": 1.05, |
| "grad_norm": 39.178640280039765, |
| "learning_rate": 8.213995280017641e-08, |
| "logits/chosen": -2.022156238555908, |
| "logits/rejected": -1.857143759727478, |
| "logps/chosen": -85.9186019897461, |
| "logps/rejected": -166.30189514160156, |
| "loss": 0.1486, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.53517484664917, |
| "rewards/margins": 4.04697322845459, |
| "rewards/rejected": -2.511798858642578, |
| "step": 2880 |
| }, |
| { |
| "epoch": 1.05, |
| "grad_norm": 34.78141019059118, |
| "learning_rate": 8.166683563796132e-08, |
| "logits/chosen": -1.9229103326797485, |
| "logits/rejected": -2.1193885803222656, |
| "logps/chosen": -80.5084228515625, |
| "logps/rejected": -117.29715728759766, |
| "loss": 0.2072, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.6038023233413696, |
| "rewards/margins": 2.8986401557922363, |
| "rewards/rejected": -1.2948377132415771, |
| "step": 2890 |
| }, |
| { |
| "epoch": 1.05, |
| "grad_norm": 55.9854636653751, |
| "learning_rate": 8.119345091042493e-08, |
| "logits/chosen": -1.9403820037841797, |
| "logits/rejected": -1.5053958892822266, |
| "logps/chosen": -78.80266571044922, |
| "logps/rejected": -152.7443389892578, |
| "loss": 0.2042, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.886501669883728, |
| "rewards/margins": 4.484723091125488, |
| "rewards/rejected": -2.59822154045105, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.05, |
| "eval_logits/chosen": -2.46332049369812, |
| "eval_logits/rejected": -2.278831958770752, |
| "eval_logps/chosen": -90.24932861328125, |
| "eval_logps/rejected": -147.7169189453125, |
| "eval_loss": 0.2783988416194916, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.9706932306289673, |
| "eval_rewards/margins": 3.205240249633789, |
| "eval_rewards/rejected": -2.2345468997955322, |
| "eval_runtime": 71.9005, |
| "eval_samples_per_second": 12.378, |
| "eval_steps_per_second": 0.195, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.06, |
| "grad_norm": 19.853743178316083, |
| "learning_rate": 8.071981761628615e-08, |
| "logits/chosen": -2.0366461277008057, |
| "logits/rejected": -1.6515051126480103, |
| "logps/chosen": -108.37039947509766, |
| "logps/rejected": -271.4614562988281, |
| "loss": 0.1636, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.959917426109314, |
| "rewards/margins": 4.299803256988525, |
| "rewards/rejected": -3.33988618850708, |
| "step": 2910 |
| }, |
| { |
| "epoch": 1.06, |
| "grad_norm": 36.91632833855024, |
| "learning_rate": 8.024595476423992e-08, |
| "logits/chosen": -1.8184095621109009, |
| "logits/rejected": -1.9267457723617554, |
| "logps/chosen": -84.40902709960938, |
| "logps/rejected": -157.05877685546875, |
| "loss": 0.1462, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5019279718399048, |
| "rewards/margins": 3.4105846881866455, |
| "rewards/rejected": -1.9086570739746094, |
| "step": 2920 |
| }, |
| { |
| "epoch": 1.06, |
| "grad_norm": 33.58138147718405, |
| "learning_rate": 7.977188137219414e-08, |
| "logits/chosen": -1.3525230884552002, |
| "logits/rejected": -1.4422528743743896, |
| "logps/chosen": -77.68660736083984, |
| "logps/rejected": -128.90638732910156, |
| "loss": 0.1572, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6080999374389648, |
| "rewards/margins": 3.3746752738952637, |
| "rewards/rejected": -1.7665754556655884, |
| "step": 2930 |
| }, |
| { |
| "epoch": 1.07, |
| "grad_norm": 32.99013154464336, |
| "learning_rate": 7.929761646650649e-08, |
| "logits/chosen": -2.245877504348755, |
| "logits/rejected": -1.9841814041137695, |
| "logps/chosen": -96.16056823730469, |
| "logps/rejected": -256.2076721191406, |
| "loss": 0.1878, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.466813564300537, |
| "rewards/margins": 4.286352157592773, |
| "rewards/rejected": -2.8195388317108154, |
| "step": 2940 |
| }, |
| { |
| "epoch": 1.07, |
| "grad_norm": 30.967043967811318, |
| "learning_rate": 7.882317908122083e-08, |
| "logits/chosen": -1.4433786869049072, |
| "logits/rejected": -1.1254138946533203, |
| "logps/chosen": -98.50581359863281, |
| "logps/rejected": -170.2156982421875, |
| "loss": 0.1753, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9996970891952515, |
| "rewards/margins": 3.900161027908325, |
| "rewards/rejected": -2.900463819503784, |
| "step": 2950 |
| }, |
| { |
| "epoch": 1.07, |
| "grad_norm": 21.50191304483852, |
| "learning_rate": 7.834858825730326e-08, |
| "logits/chosen": -1.6459989547729492, |
| "logits/rejected": -1.7681890726089478, |
| "logps/chosen": -95.52315521240234, |
| "logps/rejected": -140.78799438476562, |
| "loss": 0.1528, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.9920917749404907, |
| "rewards/margins": 3.2807719707489014, |
| "rewards/rejected": -2.2886805534362793, |
| "step": 2960 |
| }, |
| { |
| "epoch": 1.08, |
| "grad_norm": 17.260479958583524, |
| "learning_rate": 7.787386304187798e-08, |
| "logits/chosen": -1.4531595706939697, |
| "logits/rejected": -1.3701298236846924, |
| "logps/chosen": -93.404541015625, |
| "logps/rejected": -135.31427001953125, |
| "loss": 0.171, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0154047012329102, |
| "rewards/margins": 3.056318998336792, |
| "rewards/rejected": -2.040914297103882, |
| "step": 2970 |
| }, |
| { |
| "epoch": 1.08, |
| "grad_norm": 23.025875175350215, |
| "learning_rate": 7.739902248746283e-08, |
| "logits/chosen": -1.7231334447860718, |
| "logits/rejected": -1.83186936378479, |
| "logps/chosen": -107.64973449707031, |
| "logps/rejected": -145.9791717529297, |
| "loss": 0.1349, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5788373947143555, |
| "rewards/margins": 3.6469104290008545, |
| "rewards/rejected": -2.068073034286499, |
| "step": 2980 |
| }, |
| { |
| "epoch": 1.09, |
| "grad_norm": 32.06528399748813, |
| "learning_rate": 7.692408565120458e-08, |
| "logits/chosen": -2.0466766357421875, |
| "logits/rejected": -1.4406808614730835, |
| "logps/chosen": -95.41911315917969, |
| "logps/rejected": -297.7329406738281, |
| "loss": 0.1434, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.45736563205719, |
| "rewards/margins": 4.8933281898498535, |
| "rewards/rejected": -3.435962200164795, |
| "step": 2990 |
| }, |
| { |
| "epoch": 1.09, |
| "grad_norm": 31.87542686599211, |
| "learning_rate": 7.64490715941142e-08, |
| "logits/chosen": -2.2173619270324707, |
| "logits/rejected": -1.7960357666015625, |
| "logps/chosen": -85.83885192871094, |
| "logps/rejected": -169.8695831298828, |
| "loss": 0.1529, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.137475609779358, |
| "rewards/margins": 4.046895503997803, |
| "rewards/rejected": -2.9094200134277344, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.09, |
| "eval_logits/chosen": -2.4170167446136475, |
| "eval_logits/rejected": -2.2436559200286865, |
| "eval_logps/chosen": -89.03692626953125, |
| "eval_logps/rejected": -148.1836395263672, |
| "eval_loss": 0.27793529629707336, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.0919331312179565, |
| "eval_rewards/margins": 3.373152256011963, |
| "eval_rewards/rejected": -2.281219720840454, |
| "eval_runtime": 71.967, |
| "eval_samples_per_second": 12.367, |
| "eval_steps_per_second": 0.195, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.09, |
| "grad_norm": 30.683194890069508, |
| "learning_rate": 7.597399938030184e-08, |
| "logits/chosen": -1.7612870931625366, |
| "logits/rejected": -1.8522275686264038, |
| "logps/chosen": -73.18524169921875, |
| "logps/rejected": -130.78829956054688, |
| "loss": 0.1392, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7518160343170166, |
| "rewards/margins": 3.613239288330078, |
| "rewards/rejected": -1.861423134803772, |
| "step": 3010 |
| }, |
| { |
| "epoch": 1.1, |
| "grad_norm": 12.524140781131472, |
| "learning_rate": 7.549888807621168e-08, |
| "logits/chosen": -1.6050277948379517, |
| "logits/rejected": -1.5487821102142334, |
| "logps/chosen": -84.5078125, |
| "logps/rejected": -174.291259765625, |
| "loss": 0.1498, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5817402601242065, |
| "rewards/margins": 3.690995693206787, |
| "rewards/rejected": -2.10925555229187, |
| "step": 3020 |
| }, |
| { |
| "epoch": 1.1, |
| "grad_norm": 19.057459821446617, |
| "learning_rate": 7.502375674985675e-08, |
| "logits/chosen": -2.2322306632995605, |
| "logits/rejected": -1.8400142192840576, |
| "logps/chosen": -94.57080078125, |
| "logps/rejected": -167.90066528320312, |
| "loss": 0.197, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.6182721853256226, |
| "rewards/margins": 3.7063040733337402, |
| "rewards/rejected": -2.0880320072174072, |
| "step": 3030 |
| }, |
| { |
| "epoch": 1.1, |
| "grad_norm": 39.27950520182761, |
| "learning_rate": 7.454862447005359e-08, |
| "logits/chosen": -2.199615001678467, |
| "logits/rejected": -1.5867798328399658, |
| "logps/chosen": -83.38591766357422, |
| "logps/rejected": -181.61129760742188, |
| "loss": 0.1505, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.513109564781189, |
| "rewards/margins": 5.107856750488281, |
| "rewards/rejected": -3.594747543334961, |
| "step": 3040 |
| }, |
| { |
| "epoch": 1.11, |
| "grad_norm": 29.496269378804925, |
| "learning_rate": 7.407351030565711e-08, |
| "logits/chosen": -2.2386364936828613, |
| "logits/rejected": -1.7992044687271118, |
| "logps/chosen": -70.04452514648438, |
| "logps/rejected": -168.61111450195312, |
| "loss": 0.1842, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.804783582687378, |
| "rewards/margins": 4.483792781829834, |
| "rewards/rejected": -2.679008960723877, |
| "step": 3050 |
| }, |
| { |
| "epoch": 1.11, |
| "grad_norm": 28.123692890471585, |
| "learning_rate": 7.359843332479512e-08, |
| "logits/chosen": -1.5445009469985962, |
| "logits/rejected": -1.632628083229065, |
| "logps/chosen": -97.18016815185547, |
| "logps/rejected": -156.12501525878906, |
| "loss": 0.1522, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9871312975883484, |
| "rewards/margins": 3.336866855621338, |
| "rewards/rejected": -2.3497352600097656, |
| "step": 3060 |
| }, |
| { |
| "epoch": 1.11, |
| "grad_norm": 24.98966890041787, |
| "learning_rate": 7.312341259410308e-08, |
| "logits/chosen": -1.8162829875946045, |
| "logits/rejected": -1.6895732879638672, |
| "logps/chosen": -82.59261322021484, |
| "logps/rejected": -126.13545227050781, |
| "loss": 0.1533, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.7326328754425049, |
| "rewards/margins": 3.6410274505615234, |
| "rewards/rejected": -1.9083948135375977, |
| "step": 3070 |
| }, |
| { |
| "epoch": 1.12, |
| "grad_norm": 38.009951638634, |
| "learning_rate": 7.264846717795899e-08, |
| "logits/chosen": -1.9883182048797607, |
| "logits/rejected": -1.5640618801116943, |
| "logps/chosen": -79.90910339355469, |
| "logps/rejected": -204.0009765625, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5611870288848877, |
| "rewards/margins": 3.625385284423828, |
| "rewards/rejected": -2.0641980171203613, |
| "step": 3080 |
| }, |
| { |
| "epoch": 1.12, |
| "grad_norm": 37.92405502481002, |
| "learning_rate": 7.217361613771814e-08, |
| "logits/chosen": -1.5013178586959839, |
| "logits/rejected": -1.6586967706680298, |
| "logps/chosen": -74.64991760253906, |
| "logps/rejected": -133.19850158691406, |
| "loss": 0.1487, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.388861894607544, |
| "rewards/margins": 3.804910659790039, |
| "rewards/rejected": -2.416048765182495, |
| "step": 3090 |
| }, |
| { |
| "epoch": 1.13, |
| "grad_norm": 33.70546037656875, |
| "learning_rate": 7.16988785309482e-08, |
| "logits/chosen": -1.7027190923690796, |
| "logits/rejected": -1.2969590425491333, |
| "logps/chosen": -91.05853271484375, |
| "logps/rejected": -159.3774871826172, |
| "loss": 0.1675, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.82365882396698, |
| "rewards/margins": 4.186999320983887, |
| "rewards/rejected": -3.363340377807617, |
| "step": 3100 |
| }, |
| { |
| "epoch": 1.13, |
| "eval_logits/chosen": -2.3937642574310303, |
| "eval_logits/rejected": -2.2311301231384277, |
| "eval_logps/chosen": -88.40351104736328, |
| "eval_logps/rejected": -148.453857421875, |
| "eval_loss": 0.2778138816356659, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 1.155274748802185, |
| "eval_rewards/margins": 3.4635140895843506, |
| "eval_rewards/rejected": -2.308239459991455, |
| "eval_runtime": 71.9837, |
| "eval_samples_per_second": 12.364, |
| "eval_steps_per_second": 0.194, |
| "step": 3100 |
| }, |
| { |
| "epoch": 1.13, |
| "grad_norm": 10.977394963530008, |
| "learning_rate": 7.122427341066431e-08, |
| "logits/chosen": -1.7731870412826538, |
| "logits/rejected": -1.5993355512619019, |
| "logps/chosen": -94.86338806152344, |
| "logps/rejected": -159.97232055664062, |
| "loss": 0.1694, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.6271803975105286, |
| "rewards/margins": 3.4266326427459717, |
| "rewards/rejected": -2.799452304840088, |
| "step": 3110 |
| }, |
| { |
| "epoch": 1.13, |
| "grad_norm": 35.6693717697895, |
| "learning_rate": 7.074981982456438e-08, |
| "logits/chosen": -1.6707003116607666, |
| "logits/rejected": -1.3329427242279053, |
| "logps/chosen": -74.29015350341797, |
| "logps/rejected": -126.54850769042969, |
| "loss": 0.1541, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1601439714431763, |
| "rewards/margins": 3.103707790374756, |
| "rewards/rejected": -1.9435638189315796, |
| "step": 3120 |
| }, |
| { |
| "epoch": 1.14, |
| "grad_norm": 26.223529400996284, |
| "learning_rate": 7.027553681426475e-08, |
| "logits/chosen": -1.6821495294570923, |
| "logits/rejected": -1.6677749156951904, |
| "logps/chosen": -69.13279724121094, |
| "logps/rejected": -125.01466369628906, |
| "loss": 0.1206, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.598923683166504, |
| "rewards/margins": 3.550352096557617, |
| "rewards/rejected": -1.951428771018982, |
| "step": 3130 |
| }, |
| { |
| "epoch": 1.14, |
| "grad_norm": 33.00723755782217, |
| "learning_rate": 6.980144341453587e-08, |
| "logits/chosen": -1.8480640649795532, |
| "logits/rejected": -1.4870339632034302, |
| "logps/chosen": -93.90699768066406, |
| "logps/rejected": -151.79629516601562, |
| "loss": 0.1937, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.6560937166213989, |
| "rewards/margins": 3.1022205352783203, |
| "rewards/rejected": -2.446126937866211, |
| "step": 3140 |
| }, |
| { |
| "epoch": 1.14, |
| "grad_norm": 28.329720861751795, |
| "learning_rate": 6.932755865253842e-08, |
| "logits/chosen": -2.052046537399292, |
| "logits/rejected": -1.551355242729187, |
| "logps/chosen": -79.02404022216797, |
| "logps/rejected": -138.8905792236328, |
| "loss": 0.1736, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4904524087905884, |
| "rewards/margins": 3.604060411453247, |
| "rewards/rejected": -2.113607883453369, |
| "step": 3150 |
| }, |
| { |
| "epoch": 1.15, |
| "grad_norm": 22.857016307017492, |
| "learning_rate": 6.885390154705964e-08, |
| "logits/chosen": -1.932428002357483, |
| "logits/rejected": -1.4152684211730957, |
| "logps/chosen": -98.20048522949219, |
| "logps/rejected": -169.7655792236328, |
| "loss": 0.1481, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0732301473617554, |
| "rewards/margins": 4.125487327575684, |
| "rewards/rejected": -3.0522565841674805, |
| "step": 3160 |
| }, |
| { |
| "epoch": 1.15, |
| "grad_norm": 27.862330812022716, |
| "learning_rate": 6.838049110775007e-08, |
| "logits/chosen": -1.6845344305038452, |
| "logits/rejected": -1.5890744924545288, |
| "logps/chosen": -98.02180480957031, |
| "logps/rejected": -128.43356323242188, |
| "loss": 0.1365, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3146222829818726, |
| "rewards/margins": 2.931612014770508, |
| "rewards/rejected": -1.6169898509979248, |
| "step": 3170 |
| }, |
| { |
| "epoch": 1.15, |
| "grad_norm": 29.94290227789718, |
| "learning_rate": 6.790734633436058e-08, |
| "logits/chosen": -1.99330735206604, |
| "logits/rejected": -1.2950857877731323, |
| "logps/chosen": -99.69730377197266, |
| "logps/rejected": -170.60195922851562, |
| "loss": 0.1448, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1823816299438477, |
| "rewards/margins": 3.796720504760742, |
| "rewards/rejected": -2.6143388748168945, |
| "step": 3180 |
| }, |
| { |
| "epoch": 1.16, |
| "grad_norm": 64.10187404062623, |
| "learning_rate": 6.743448621597989e-08, |
| "logits/chosen": -1.8239631652832031, |
| "logits/rejected": -1.8418114185333252, |
| "logps/chosen": -100.65877532958984, |
| "logps/rejected": -151.15411376953125, |
| "loss": 0.131, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8053275942802429, |
| "rewards/margins": 3.210911989212036, |
| "rewards/rejected": -2.4055840969085693, |
| "step": 3190 |
| }, |
| { |
| "epoch": 1.16, |
| "grad_norm": 26.247231746645152, |
| "learning_rate": 6.696192973027241e-08, |
| "logits/chosen": -2.037968158721924, |
| "logits/rejected": -1.6115257740020752, |
| "logps/chosen": -73.27265930175781, |
| "logps/rejected": -159.865966796875, |
| "loss": 0.1542, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.6886720657348633, |
| "rewards/margins": 3.8989975452423096, |
| "rewards/rejected": -2.2103257179260254, |
| "step": 3200 |
| }, |
| { |
| "epoch": 1.16, |
| "eval_logits/chosen": -2.4345650672912598, |
| "eval_logits/rejected": -2.2589755058288574, |
| "eval_logps/chosen": -90.60651397705078, |
| "eval_logps/rejected": -151.0882110595703, |
| "eval_loss": 0.27637749910354614, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 0.9349749684333801, |
| "eval_rewards/margins": 3.50665283203125, |
| "eval_rewards/rejected": -2.5716776847839355, |
| "eval_runtime": 71.9022, |
| "eval_samples_per_second": 12.378, |
| "eval_steps_per_second": 0.195, |
| "step": 3200 |
| }, |
| { |
| "epoch": 1.17, |
| "grad_norm": 26.043972128745228, |
| "learning_rate": 6.648969584271668e-08, |
| "logits/chosen": -1.7389099597930908, |
| "logits/rejected": -1.7843681573867798, |
| "logps/chosen": -92.87619018554688, |
| "logps/rejected": -143.67486572265625, |
| "loss": 0.155, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.270666241645813, |
| "rewards/margins": 3.2985167503356934, |
| "rewards/rejected": -2.0278501510620117, |
| "step": 3210 |
| }, |
| { |
| "epoch": 1.17, |
| "grad_norm": 48.1935636258946, |
| "learning_rate": 6.601780350584408e-08, |
| "logits/chosen": -2.341850757598877, |
| "logits/rejected": -2.0155091285705566, |
| "logps/chosen": -91.69564819335938, |
| "logps/rejected": -173.34188842773438, |
| "loss": 0.1687, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.45611572265625, |
| "rewards/margins": 4.184123516082764, |
| "rewards/rejected": -2.7280075550079346, |
| "step": 3220 |
| }, |
| { |
| "epoch": 1.17, |
| "grad_norm": 23.817770856670688, |
| "learning_rate": 6.55462716584783e-08, |
| "logits/chosen": -1.6578280925750732, |
| "logits/rejected": -1.547189712524414, |
| "logps/chosen": -70.36898803710938, |
| "logps/rejected": -163.3883819580078, |
| "loss": 0.1816, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.5254313945770264, |
| "rewards/margins": 4.97360897064209, |
| "rewards/rejected": -3.4481780529022217, |
| "step": 3230 |
| }, |
| { |
| "epoch": 1.18, |
| "grad_norm": 30.43724197841503, |
| "learning_rate": 6.507511922497525e-08, |
| "logits/chosen": -1.9906114339828491, |
| "logits/rejected": -1.7292976379394531, |
| "logps/chosen": -70.70221710205078, |
| "logps/rejected": -145.86181640625, |
| "loss": 0.1662, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.5770456790924072, |
| "rewards/margins": 4.609046459197998, |
| "rewards/rejected": -3.03200101852417, |
| "step": 3240 |
| }, |
| { |
| "epoch": 1.18, |
| "grad_norm": 30.164526587495324, |
| "learning_rate": 6.460436511446348e-08, |
| "logits/chosen": -1.9928280115127563, |
| "logits/rejected": -1.7679364681243896, |
| "logps/chosen": -100.5902328491211, |
| "logps/rejected": -178.82383728027344, |
| "loss": 0.1463, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6832008361816406, |
| "rewards/margins": 4.548179626464844, |
| "rewards/rejected": -3.864978790283203, |
| "step": 3250 |
| }, |
| { |
| "epoch": 1.18, |
| "grad_norm": 52.18169505988799, |
| "learning_rate": 6.413402822008541e-08, |
| "logits/chosen": -2.0655717849731445, |
| "logits/rejected": -1.3271939754486084, |
| "logps/chosen": -87.31197357177734, |
| "logps/rejected": -160.46385192871094, |
| "loss": 0.1905, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.8497316241264343, |
| "rewards/margins": 3.820449113845825, |
| "rewards/rejected": -2.9707179069519043, |
| "step": 3260 |
| }, |
| { |
| "epoch": 1.19, |
| "grad_norm": 28.76843973839471, |
| "learning_rate": 6.366412741823888e-08, |
| "logits/chosen": -1.6622741222381592, |
| "logits/rejected": -1.776906967163086, |
| "logps/chosen": -94.9710693359375, |
| "logps/rejected": -128.8150634765625, |
| "loss": 0.1917, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5094915628433228, |
| "rewards/margins": 3.212104082107544, |
| "rewards/rejected": -1.7026126384735107, |
| "step": 3270 |
| }, |
| { |
| "epoch": 1.19, |
| "grad_norm": 29.1350717468671, |
| "learning_rate": 6.31946815678198e-08, |
| "logits/chosen": -2.0058090686798096, |
| "logits/rejected": -1.7673218250274658, |
| "logps/chosen": -95.3953857421875, |
| "logps/rejected": -176.0774688720703, |
| "loss": 0.1337, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4662463665008545, |
| "rewards/margins": 4.142214775085449, |
| "rewards/rejected": -2.6759676933288574, |
| "step": 3280 |
| }, |
| { |
| "epoch": 1.19, |
| "grad_norm": 27.79208712883472, |
| "learning_rate": 6.272570950946508e-08, |
| "logits/chosen": -2.106937885284424, |
| "logits/rejected": -1.6478996276855469, |
| "logps/chosen": -85.5794906616211, |
| "logps/rejected": -173.1703338623047, |
| "loss": 0.1716, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1876065731048584, |
| "rewards/margins": 4.5130767822265625, |
| "rewards/rejected": -3.325469970703125, |
| "step": 3290 |
| }, |
| { |
| "epoch": 1.2, |
| "grad_norm": 26.268117829624686, |
| "learning_rate": 6.225723006479663e-08, |
| "logits/chosen": -2.263885259628296, |
| "logits/rejected": -1.6747421026229858, |
| "logps/chosen": -63.94829559326172, |
| "logps/rejected": -163.29486083984375, |
| "loss": 0.1694, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9152793884277344, |
| "rewards/margins": 4.830869197845459, |
| "rewards/rejected": -2.9155895709991455, |
| "step": 3300 |
| }, |
| { |
| "epoch": 1.2, |
| "eval_logits/chosen": -2.4307594299316406, |
| "eval_logits/rejected": -2.257986545562744, |
| "eval_logps/chosen": -91.0833969116211, |
| "eval_logps/rejected": -151.17442321777344, |
| "eval_loss": 0.27279049158096313, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.887286365032196, |
| "eval_rewards/margins": 3.4675843715667725, |
| "eval_rewards/rejected": -2.5802979469299316, |
| "eval_runtime": 71.844, |
| "eval_samples_per_second": 12.388, |
| "eval_steps_per_second": 0.195, |
| "step": 3300 |
| }, |
| { |
| "epoch": 1.2, |
| "grad_norm": 20.771596605110865, |
| "learning_rate": 6.178926203566588e-08, |
| "logits/chosen": -1.9085609912872314, |
| "logits/rejected": -1.6423466205596924, |
| "logps/chosen": -88.54236602783203, |
| "logps/rejected": -151.7003631591797, |
| "loss": 0.1747, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4402354955673218, |
| "rewards/margins": 4.408102989196777, |
| "rewards/rejected": -2.9678680896759033, |
| "step": 3310 |
| }, |
| { |
| "epoch": 1.21, |
| "grad_norm": 23.09159131359664, |
| "learning_rate": 6.132182420339918e-08, |
| "logits/chosen": -1.6666587591171265, |
| "logits/rejected": -1.2324297428131104, |
| "logps/chosen": -81.8795166015625, |
| "logps/rejected": -138.54940795898438, |
| "loss": 0.1441, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4413508176803589, |
| "rewards/margins": 3.961609363555908, |
| "rewards/rejected": -2.5202584266662598, |
| "step": 3320 |
| }, |
| { |
| "epoch": 1.21, |
| "grad_norm": 33.84562565815576, |
| "learning_rate": 6.085493532804413e-08, |
| "logits/chosen": -2.300516128540039, |
| "logits/rejected": -2.122788906097412, |
| "logps/chosen": -86.87980651855469, |
| "logps/rejected": -168.46022033691406, |
| "loss": 0.1544, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0005496740341187, |
| "rewards/margins": 3.9717116355895996, |
| "rewards/rejected": -2.9711620807647705, |
| "step": 3330 |
| }, |
| { |
| "epoch": 1.21, |
| "grad_norm": 14.408331018978265, |
| "learning_rate": 6.03886141476166e-08, |
| "logits/chosen": -2.2469568252563477, |
| "logits/rejected": -1.8382642269134521, |
| "logps/chosen": -81.93803405761719, |
| "logps/rejected": -163.33473205566406, |
| "loss": 0.1334, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7893930673599243, |
| "rewards/margins": 4.628080368041992, |
| "rewards/rejected": -2.8386874198913574, |
| "step": 3340 |
| }, |
| { |
| "epoch": 1.22, |
| "grad_norm": 33.15124696775361, |
| "learning_rate": 5.992287937734873e-08, |
| "logits/chosen": -2.054621934890747, |
| "logits/rejected": -1.8777787685394287, |
| "logps/chosen": -84.07951354980469, |
| "logps/rejected": -179.1838836669922, |
| "loss": 0.144, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.4575679302215576, |
| "rewards/margins": 4.167536735534668, |
| "rewards/rejected": -2.7099688053131104, |
| "step": 3350 |
| }, |
| { |
| "epoch": 1.22, |
| "grad_norm": 32.145978602477854, |
| "learning_rate": 5.9457749708937756e-08, |
| "logits/chosen": -1.9643110036849976, |
| "logits/rejected": -1.795340895652771, |
| "logps/chosen": -82.72921752929688, |
| "logps/rejected": -128.0702667236328, |
| "loss": 0.1543, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9123824238777161, |
| "rewards/margins": 3.376009464263916, |
| "rewards/rejected": -2.4636270999908447, |
| "step": 3360 |
| }, |
| { |
| "epoch": 1.22, |
| "grad_norm": 49.92991955367269, |
| "learning_rate": 5.8993243809795915e-08, |
| "logits/chosen": -1.8667415380477905, |
| "logits/rejected": -1.3340458869934082, |
| "logps/chosen": -91.13977813720703, |
| "logps/rejected": -158.5907440185547, |
| "loss": 0.1547, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.49219757318496704, |
| "rewards/margins": 3.9671154022216797, |
| "rewards/rejected": -3.4749176502227783, |
| "step": 3370 |
| }, |
| { |
| "epoch": 1.23, |
| "grad_norm": 31.770406094051957, |
| "learning_rate": 5.852938032230126e-08, |
| "logits/chosen": -2.3319461345672607, |
| "logits/rejected": -1.882067322731018, |
| "logps/chosen": -76.39924621582031, |
| "logps/rejected": -176.21929931640625, |
| "loss": 0.1699, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.2963407039642334, |
| "rewards/margins": 6.4765625, |
| "rewards/rejected": -4.180222034454346, |
| "step": 3380 |
| }, |
| { |
| "epoch": 1.23, |
| "grad_norm": 21.3080930046784, |
| "learning_rate": 5.806617786304937e-08, |
| "logits/chosen": -1.8695051670074463, |
| "logits/rejected": -1.3510863780975342, |
| "logps/chosen": -86.91441345214844, |
| "logps/rejected": -250.5083770751953, |
| "loss": 0.1788, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.846575140953064, |
| "rewards/margins": 5.218047142028809, |
| "rewards/rejected": -4.371471405029297, |
| "step": 3390 |
| }, |
| { |
| "epoch": 1.23, |
| "grad_norm": 41.88504763461129, |
| "learning_rate": 5.760365502210634e-08, |
| "logits/chosen": -1.7990360260009766, |
| "logits/rejected": -1.9284346103668213, |
| "logps/chosen": -91.82389068603516, |
| "logps/rejected": -152.16659545898438, |
| "loss": 0.1763, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4950382709503174, |
| "rewards/margins": 4.346599578857422, |
| "rewards/rejected": -2.8515613079071045, |
| "step": 3400 |
| }, |
| { |
| "epoch": 1.23, |
| "eval_logits/chosen": -2.414144277572632, |
| "eval_logits/rejected": -2.243586778640747, |
| "eval_logps/chosen": -91.0235366821289, |
| "eval_logps/rejected": -152.044921875, |
| "eval_loss": 0.26993635296821594, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.8932717442512512, |
| "eval_rewards/margins": 3.5606191158294678, |
| "eval_rewards/rejected": -2.6673471927642822, |
| "eval_runtime": 71.993, |
| "eval_samples_per_second": 12.362, |
| "eval_steps_per_second": 0.194, |
| "step": 3400 |
| }, |
| { |
| "epoch": 1.24, |
| "grad_norm": 25.05732311627679, |
| "learning_rate": 5.7141830362262514e-08, |
| "logits/chosen": -1.7883466482162476, |
| "logits/rejected": -1.5041227340698242, |
| "logps/chosen": -86.22000885009766, |
| "logps/rejected": -157.66397094726562, |
| "loss": 0.1676, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.7772796154022217, |
| "rewards/margins": 4.851300239562988, |
| "rewards/rejected": -3.0740203857421875, |
| "step": 3410 |
| }, |
| { |
| "epoch": 1.24, |
| "grad_norm": 44.30312099667292, |
| "learning_rate": 5.6680722418287674e-08, |
| "logits/chosen": -2.064953565597534, |
| "logits/rejected": -1.8630918264389038, |
| "logps/chosen": -60.13167190551758, |
| "logps/rejected": -247.2148895263672, |
| "loss": 0.1305, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.1393609046936035, |
| "rewards/margins": 4.484022617340088, |
| "rewards/rejected": -2.3446614742279053, |
| "step": 3420 |
| }, |
| { |
| "epoch": 1.25, |
| "grad_norm": 48.64244001308947, |
| "learning_rate": 5.622034969618704e-08, |
| "logits/chosen": -1.976629614830017, |
| "logits/rejected": -1.4999226331710815, |
| "logps/chosen": -77.93717956542969, |
| "logps/rejected": -151.5218963623047, |
| "loss": 0.1785, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5172656774520874, |
| "rewards/margins": 3.7420005798339844, |
| "rewards/rejected": -2.2247347831726074, |
| "step": 3430 |
| }, |
| { |
| "epoch": 1.25, |
| "grad_norm": 35.51662255328931, |
| "learning_rate": 5.576073067245862e-08, |
| "logits/chosen": -1.8411935567855835, |
| "logits/rejected": -1.700330376625061, |
| "logps/chosen": -67.32380676269531, |
| "logps/rejected": -140.62051391601562, |
| "loss": 0.167, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6732587814331055, |
| "rewards/margins": 4.312959671020508, |
| "rewards/rejected": -2.6397011280059814, |
| "step": 3440 |
| }, |
| { |
| "epoch": 1.25, |
| "grad_norm": 42.40486506801471, |
| "learning_rate": 5.530188379335166e-08, |
| "logits/chosen": -1.5920283794403076, |
| "logits/rejected": -1.1166603565216064, |
| "logps/chosen": -70.99920654296875, |
| "logps/rejected": -169.20806884765625, |
| "loss": 0.1756, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8735821843147278, |
| "rewards/margins": 5.562444686889648, |
| "rewards/rejected": -4.6888628005981445, |
| "step": 3450 |
| }, |
| { |
| "epoch": 1.26, |
| "grad_norm": 26.913972282878095, |
| "learning_rate": 5.4843827474126274e-08, |
| "logits/chosen": -1.9140102863311768, |
| "logits/rejected": -1.400821328163147, |
| "logps/chosen": -74.26724243164062, |
| "logps/rejected": -132.2233123779297, |
| "loss": 0.1374, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5520453453063965, |
| "rewards/margins": 4.453476905822754, |
| "rewards/rejected": -2.90143084526062, |
| "step": 3460 |
| }, |
| { |
| "epoch": 1.26, |
| "grad_norm": 26.13373331897645, |
| "learning_rate": 5.438658009831448e-08, |
| "logits/chosen": -1.73845636844635, |
| "logits/rejected": -1.6531593799591064, |
| "logps/chosen": -96.46147918701172, |
| "logps/rejected": -160.9883575439453, |
| "loss": 0.1403, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2454642057418823, |
| "rewards/margins": 4.1239237785339355, |
| "rewards/rejected": -2.8784596920013428, |
| "step": 3470 |
| }, |
| { |
| "epoch": 1.26, |
| "grad_norm": 34.42235214848031, |
| "learning_rate": 5.39301600169823e-08, |
| "logits/chosen": -1.9057807922363281, |
| "logits/rejected": -1.927220344543457, |
| "logps/chosen": -115.20938873291016, |
| "logps/rejected": -181.9356231689453, |
| "loss": 0.1796, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7145345211029053, |
| "rewards/margins": 4.270758152008057, |
| "rewards/rejected": -3.5562233924865723, |
| "step": 3480 |
| }, |
| { |
| "epoch": 1.27, |
| "grad_norm": 20.971370688650854, |
| "learning_rate": 5.347458554799332e-08, |
| "logits/chosen": -1.615875005722046, |
| "logits/rejected": -1.6177091598510742, |
| "logps/chosen": -118.13121032714844, |
| "logps/rejected": -160.26910400390625, |
| "loss": 0.163, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0960075855255127, |
| "rewards/margins": 3.6775288581848145, |
| "rewards/rejected": -2.581521511077881, |
| "step": 3490 |
| }, |
| { |
| "epoch": 1.27, |
| "grad_norm": 27.13239221195734, |
| "learning_rate": 5.301987497527353e-08, |
| "logits/chosen": -2.2539658546447754, |
| "logits/rejected": -1.4924699068069458, |
| "logps/chosen": -91.34033203125, |
| "logps/rejected": -181.88430786132812, |
| "loss": 0.1526, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.448373556137085, |
| "rewards/margins": 4.606696128845215, |
| "rewards/rejected": -3.1583220958709717, |
| "step": 3500 |
| }, |
| { |
| "epoch": 1.27, |
| "eval_logits/chosen": -2.421151876449585, |
| "eval_logits/rejected": -2.2513325214385986, |
| "eval_logps/chosen": -91.84569549560547, |
| "eval_logps/rejected": -153.45870971679688, |
| "eval_loss": 0.2666407823562622, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 0.8110561370849609, |
| "eval_rewards/margins": 3.6197829246520996, |
| "eval_rewards/rejected": -2.8087267875671387, |
| "eval_runtime": 71.9717, |
| "eval_samples_per_second": 12.366, |
| "eval_steps_per_second": 0.195, |
| "step": 3500 |
| }, |
| { |
| "epoch": 1.27, |
| "grad_norm": 29.49364758024353, |
| "learning_rate": 5.256604654807742e-08, |
| "logits/chosen": -1.7953096628189087, |
| "logits/rejected": -1.669303297996521, |
| "logps/chosen": -90.09492492675781, |
| "logps/rejected": -149.954833984375, |
| "loss": 0.1912, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0876281261444092, |
| "rewards/margins": 3.9230809211730957, |
| "rewards/rejected": -2.8354527950286865, |
| "step": 3510 |
| }, |
| { |
| "epoch": 1.28, |
| "grad_norm": 41.572390412456194, |
| "learning_rate": 5.21131184802557e-08, |
| "logits/chosen": -2.22542142868042, |
| "logits/rejected": -1.8966829776763916, |
| "logps/chosen": -83.38715362548828, |
| "logps/rejected": -141.80059814453125, |
| "loss": 0.1877, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1568256616592407, |
| "rewards/margins": 3.453406572341919, |
| "rewards/rejected": -2.2965807914733887, |
| "step": 3520 |
| }, |
| { |
| "epoch": 1.28, |
| "grad_norm": 24.565593605623125, |
| "learning_rate": 5.166110894952426e-08, |
| "logits/chosen": -1.9109958410263062, |
| "logits/rejected": -1.4489656686782837, |
| "logps/chosen": -58.008819580078125, |
| "logps/rejected": -191.261474609375, |
| "loss": 0.1478, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.9802272319793701, |
| "rewards/margins": 6.276611804962158, |
| "rewards/rejected": -4.296384811401367, |
| "step": 3530 |
| }, |
| { |
| "epoch": 1.28, |
| "grad_norm": 53.61199506683095, |
| "learning_rate": 5.1210036096734595e-08, |
| "logits/chosen": -1.5364563465118408, |
| "logits/rejected": -1.3872387409210205, |
| "logps/chosen": -107.84410095214844, |
| "logps/rejected": -160.59361267089844, |
| "loss": 0.1401, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.5507475137710571, |
| "rewards/margins": 3.1839842796325684, |
| "rewards/rejected": -2.633236885070801, |
| "step": 3540 |
| }, |
| { |
| "epoch": 1.29, |
| "grad_norm": 28.74282956678025, |
| "learning_rate": 5.0759918025145814e-08, |
| "logits/chosen": -2.2165632247924805, |
| "logits/rejected": -1.650313138961792, |
| "logps/chosen": -93.6681137084961, |
| "logps/rejected": -145.92111206054688, |
| "loss": 0.1582, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3355414867401123, |
| "rewards/margins": 4.038471698760986, |
| "rewards/rejected": -2.702929973602295, |
| "step": 3550 |
| }, |
| { |
| "epoch": 1.29, |
| "grad_norm": 26.165374741112064, |
| "learning_rate": 5.031077279969797e-08, |
| "logits/chosen": -1.9472557306289673, |
| "logits/rejected": -1.7647784948349, |
| "logps/chosen": -76.48893737792969, |
| "logps/rejected": -135.52386474609375, |
| "loss": 0.1404, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.397589921951294, |
| "rewards/margins": 4.449340343475342, |
| "rewards/rejected": -3.051750898361206, |
| "step": 3560 |
| }, |
| { |
| "epoch": 1.3, |
| "grad_norm": 36.82965132004594, |
| "learning_rate": 4.9862618446287206e-08, |
| "logits/chosen": -1.9602140188217163, |
| "logits/rejected": -1.6700010299682617, |
| "logps/chosen": -72.4864730834961, |
| "logps/rejected": -136.77584838867188, |
| "loss": 0.171, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.6417019367218018, |
| "rewards/margins": 3.702420473098755, |
| "rewards/rejected": -2.060718536376953, |
| "step": 3570 |
| }, |
| { |
| "epoch": 1.3, |
| "grad_norm": 24.181445490171278, |
| "learning_rate": 4.9415472951042175e-08, |
| "logits/chosen": -1.9628846645355225, |
| "logits/rejected": -1.5444905757904053, |
| "logps/chosen": -81.23819732666016, |
| "logps/rejected": -182.43032836914062, |
| "loss": 0.121, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.530139684677124, |
| "rewards/margins": 4.815797328948975, |
| "rewards/rejected": -3.2856578826904297, |
| "step": 3580 |
| }, |
| { |
| "epoch": 1.3, |
| "grad_norm": 45.8781810453152, |
| "learning_rate": 4.8969354259602245e-08, |
| "logits/chosen": -1.7887170314788818, |
| "logits/rejected": -1.6762058734893799, |
| "logps/chosen": -87.67835998535156, |
| "logps/rejected": -148.01345825195312, |
| "loss": 0.1509, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3987677097320557, |
| "rewards/margins": 4.317151069641113, |
| "rewards/rejected": -2.9183835983276367, |
| "step": 3590 |
| }, |
| { |
| "epoch": 1.31, |
| "grad_norm": 46.13081539436112, |
| "learning_rate": 4.85242802763973e-08, |
| "logits/chosen": -1.9573310613632202, |
| "logits/rejected": -1.7241952419281006, |
| "logps/chosen": -68.08113861083984, |
| "logps/rejected": -126.60243225097656, |
| "loss": 0.1819, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.9216814041137695, |
| "rewards/margins": 4.243165969848633, |
| "rewards/rejected": -2.321484327316284, |
| "step": 3600 |
| }, |
| { |
| "epoch": 1.31, |
| "eval_logits/chosen": -2.4482152462005615, |
| "eval_logits/rejected": -2.2676963806152344, |
| "eval_logps/chosen": -91.19036865234375, |
| "eval_logps/rejected": -152.58682250976562, |
| "eval_loss": 0.26565536856651306, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.8765901327133179, |
| "eval_rewards/margins": 3.5981290340423584, |
| "eval_rewards/rejected": -2.721538782119751, |
| "eval_runtime": 71.9607, |
| "eval_samples_per_second": 12.368, |
| "eval_steps_per_second": 0.195, |
| "step": 3600 |
| }, |
| { |
| "epoch": 1.31, |
| "grad_norm": 23.242650326252527, |
| "learning_rate": 4.808026886392907e-08, |
| "logits/chosen": -1.5748409032821655, |
| "logits/rejected": -1.6680151224136353, |
| "logps/chosen": -97.38771057128906, |
| "logps/rejected": -131.87599182128906, |
| "loss": 0.1103, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.183221459388733, |
| "rewards/margins": 2.838090419769287, |
| "rewards/rejected": -1.654868721961975, |
| "step": 3610 |
| }, |
| { |
| "epoch": 1.31, |
| "grad_norm": 20.775972068583737, |
| "learning_rate": 4.763733784205434e-08, |
| "logits/chosen": -2.157090902328491, |
| "logits/rejected": -1.7526382207870483, |
| "logps/chosen": -83.79398345947266, |
| "logps/rejected": -138.4102020263672, |
| "loss": 0.1308, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2044012546539307, |
| "rewards/margins": 3.7157578468322754, |
| "rewards/rejected": -2.5113563537597656, |
| "step": 3620 |
| }, |
| { |
| "epoch": 1.32, |
| "grad_norm": 27.624631894217778, |
| "learning_rate": 4.7195504987269736e-08, |
| "logits/chosen": -1.8666884899139404, |
| "logits/rejected": -1.962099313735962, |
| "logps/chosen": -92.00392150878906, |
| "logps/rejected": -265.90936279296875, |
| "loss": 0.1808, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.662792444229126, |
| "rewards/margins": 4.868588924407959, |
| "rewards/rejected": -3.205796718597412, |
| "step": 3630 |
| }, |
| { |
| "epoch": 1.32, |
| "grad_norm": 35.3219065228648, |
| "learning_rate": 4.6754788031998294e-08, |
| "logits/chosen": -2.1062629222869873, |
| "logits/rejected": -1.9411357641220093, |
| "logps/chosen": -81.11627960205078, |
| "logps/rejected": -207.1051483154297, |
| "loss": 0.1747, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1783745288848877, |
| "rewards/margins": 4.6277360916137695, |
| "rewards/rejected": -3.4493613243103027, |
| "step": 3640 |
| }, |
| { |
| "epoch": 1.32, |
| "grad_norm": 15.460191475491241, |
| "learning_rate": 4.631520466387777e-08, |
| "logits/chosen": -1.867004632949829, |
| "logits/rejected": -1.6915152072906494, |
| "logps/chosen": -110.6289291381836, |
| "logps/rejected": -156.9150390625, |
| "loss": 0.1242, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.2745405435562134, |
| "rewards/margins": 3.7722878456115723, |
| "rewards/rejected": -2.4977474212646484, |
| "step": 3650 |
| }, |
| { |
| "epoch": 1.33, |
| "grad_norm": 35.15290230831595, |
| "learning_rate": 4.587677252505077e-08, |
| "logits/chosen": -1.8647207021713257, |
| "logits/rejected": -1.6314834356307983, |
| "logps/chosen": -82.48086547851562, |
| "logps/rejected": -158.58407592773438, |
| "loss": 0.1358, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5844959020614624, |
| "rewards/margins": 4.390726566314697, |
| "rewards/rejected": -2.806230068206787, |
| "step": 3660 |
| }, |
| { |
| "epoch": 1.33, |
| "grad_norm": 31.64353347707315, |
| "learning_rate": 4.5439509211456734e-08, |
| "logits/chosen": -2.0931248664855957, |
| "logits/rejected": -1.6264079809188843, |
| "logps/chosen": -62.890602111816406, |
| "logps/rejected": -150.46688842773438, |
| "loss": 0.1504, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1719433069229126, |
| "rewards/margins": 4.196671009063721, |
| "rewards/rejected": -3.0247273445129395, |
| "step": 3670 |
| }, |
| { |
| "epoch": 1.34, |
| "grad_norm": 14.71173714786521, |
| "learning_rate": 4.500343227212572e-08, |
| "logits/chosen": -1.7733246088027954, |
| "logits/rejected": -2.041614294052124, |
| "logps/chosen": -123.92191314697266, |
| "logps/rejected": -155.9501953125, |
| "loss": 0.1176, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.6424921154975891, |
| "rewards/margins": 3.105193853378296, |
| "rewards/rejected": -2.4627020359039307, |
| "step": 3680 |
| }, |
| { |
| "epoch": 1.34, |
| "grad_norm": 24.247587972420103, |
| "learning_rate": 4.4568559208474127e-08, |
| "logits/chosen": -1.6494388580322266, |
| "logits/rejected": -1.4123470783233643, |
| "logps/chosen": -117.4607162475586, |
| "logps/rejected": -177.09315490722656, |
| "loss": 0.1495, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.41657859086990356, |
| "rewards/margins": 4.236758232116699, |
| "rewards/rejected": -3.8201801776885986, |
| "step": 3690 |
| }, |
| { |
| "epoch": 1.34, |
| "grad_norm": 26.64386939815121, |
| "learning_rate": 4.4134907473602205e-08, |
| "logits/chosen": -2.0543887615203857, |
| "logits/rejected": -1.703619360923767, |
| "logps/chosen": -76.43885803222656, |
| "logps/rejected": -196.35208129882812, |
| "loss": 0.1192, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.2960059642791748, |
| "rewards/margins": 4.0435991287231445, |
| "rewards/rejected": -2.747593641281128, |
| "step": 3700 |
| }, |
| { |
| "epoch": 1.34, |
| "eval_logits/chosen": -2.443880319595337, |
| "eval_logits/rejected": -2.264932155609131, |
| "eval_logps/chosen": -91.3131332397461, |
| "eval_logps/rejected": -153.05369567871094, |
| "eval_loss": 0.26342639327049255, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.8643126487731934, |
| "eval_rewards/margins": 3.6325364112854004, |
| "eval_rewards/rejected": -2.768224000930786, |
| "eval_runtime": 71.8523, |
| "eval_samples_per_second": 12.387, |
| "eval_steps_per_second": 0.195, |
| "step": 3700 |
| }, |
| { |
| "epoch": 1.35, |
| "grad_norm": 40.03721241840345, |
| "learning_rate": 4.370249447159372e-08, |
| "logits/chosen": -1.892690896987915, |
| "logits/rejected": -1.5442091226577759, |
| "logps/chosen": -105.28565979003906, |
| "logps/rejected": -149.9592742919922, |
| "loss": 0.1496, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.7859503030776978, |
| "rewards/margins": 3.7989330291748047, |
| "rewards/rejected": -3.0129826068878174, |
| "step": 3710 |
| }, |
| { |
| "epoch": 1.35, |
| "grad_norm": 35.14173667461832, |
| "learning_rate": 4.32713375568174e-08, |
| "logits/chosen": -1.6806223392486572, |
| "logits/rejected": -1.7743151187896729, |
| "logps/chosen": -96.13143157958984, |
| "logps/rejected": -149.65716552734375, |
| "loss": 0.1656, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.204106092453003, |
| "rewards/margins": 3.2453396320343018, |
| "rewards/rejected": -2.041234016418457, |
| "step": 3720 |
| }, |
| { |
| "epoch": 1.35, |
| "grad_norm": 18.483200235512868, |
| "learning_rate": 4.284145403323043e-08, |
| "logits/chosen": -1.508544683456421, |
| "logits/rejected": -1.6680357456207275, |
| "logps/chosen": -111.27012634277344, |
| "logps/rejected": -159.14892578125, |
| "loss": 0.1446, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.998451828956604, |
| "rewards/margins": 4.472312927246094, |
| "rewards/rejected": -3.4738609790802, |
| "step": 3730 |
| }, |
| { |
| "epoch": 1.36, |
| "grad_norm": 36.84141617747612, |
| "learning_rate": 4.2412861153684e-08, |
| "logits/chosen": -1.8184179067611694, |
| "logits/rejected": -1.6679880619049072, |
| "logps/chosen": -99.74363708496094, |
| "logps/rejected": -159.52243041992188, |
| "loss": 0.1462, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8888320922851562, |
| "rewards/margins": 3.3356125354766846, |
| "rewards/rejected": -2.4467806816101074, |
| "step": 3740 |
| }, |
| { |
| "epoch": 1.36, |
| "grad_norm": 20.65082234776276, |
| "learning_rate": 4.198557611923083e-08, |
| "logits/chosen": -1.5405653715133667, |
| "logits/rejected": -1.5137519836425781, |
| "logps/chosen": -83.92024230957031, |
| "logps/rejected": -171.1388702392578, |
| "loss": 0.1492, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2103972434997559, |
| "rewards/margins": 5.250774383544922, |
| "rewards/rejected": -4.040377616882324, |
| "step": 3750 |
| }, |
| { |
| "epoch": 1.36, |
| "grad_norm": 25.88209046742063, |
| "learning_rate": 4.1559616078434946e-08, |
| "logits/chosen": -2.0902841091156006, |
| "logits/rejected": -1.6206400394439697, |
| "logps/chosen": -79.99602508544922, |
| "logps/rejected": -223.35891723632812, |
| "loss": 0.1586, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2605037689208984, |
| "rewards/margins": 4.679810047149658, |
| "rewards/rejected": -3.419306993484497, |
| "step": 3760 |
| }, |
| { |
| "epoch": 1.37, |
| "grad_norm": 31.138553957115032, |
| "learning_rate": 4.113499812668331e-08, |
| "logits/chosen": -2.0036697387695312, |
| "logits/rejected": -1.8283746242523193, |
| "logps/chosen": -75.56672668457031, |
| "logps/rejected": -130.79147338867188, |
| "loss": 0.1369, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.610881507396698, |
| "rewards/margins": 3.6262385845184326, |
| "rewards/rejected": -3.015357255935669, |
| "step": 3770 |
| }, |
| { |
| "epoch": 1.37, |
| "grad_norm": 22.960729088747243, |
| "learning_rate": 4.071173930549979e-08, |
| "logits/chosen": -1.7299280166625977, |
| "logits/rejected": -1.4976304769515991, |
| "logps/chosen": -94.44389343261719, |
| "logps/rejected": -154.29798889160156, |
| "loss": 0.1498, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5378260612487793, |
| "rewards/margins": 3.7743492126464844, |
| "rewards/rejected": -3.236523151397705, |
| "step": 3780 |
| }, |
| { |
| "epoch": 1.38, |
| "grad_norm": 28.61684578895429, |
| "learning_rate": 4.0289856601861285e-08, |
| "logits/chosen": -1.8740425109863281, |
| "logits/rejected": -1.5999120473861694, |
| "logps/chosen": -87.98422241210938, |
| "logps/rejected": -153.87075805664062, |
| "loss": 0.1542, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.4116048216819763, |
| "rewards/margins": 3.741361141204834, |
| "rewards/rejected": -3.32975697517395, |
| "step": 3790 |
| }, |
| { |
| "epoch": 1.38, |
| "grad_norm": 35.76212329882805, |
| "learning_rate": 3.9869366947515747e-08, |
| "logits/chosen": -2.0777387619018555, |
| "logits/rejected": -1.9436269998550415, |
| "logps/chosen": -91.76081848144531, |
| "logps/rejected": -149.74957275390625, |
| "loss": 0.144, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.5345284938812256, |
| "rewards/margins": 4.457833290100098, |
| "rewards/rejected": -2.923304557800293, |
| "step": 3800 |
| }, |
| { |
| "epoch": 1.38, |
| "eval_logits/chosen": -2.445099115371704, |
| "eval_logits/rejected": -2.2687337398529053, |
| "eval_logps/chosen": -92.02857971191406, |
| "eval_logps/rejected": -154.06690979003906, |
| "eval_loss": 0.263904869556427, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7927670478820801, |
| "eval_rewards/margins": 3.662313938140869, |
| "eval_rewards/rejected": -2.869547128677368, |
| "eval_runtime": 72.6671, |
| "eval_samples_per_second": 12.248, |
| "eval_steps_per_second": 0.193, |
| "step": 3800 |
| }, |
| { |
| "epoch": 1.38, |
| "grad_norm": 34.70092608089086, |
| "learning_rate": 3.945028721830289e-08, |
| "logits/chosen": -1.8025611639022827, |
| "logits/rejected": -1.4805386066436768, |
| "logps/chosen": -94.7345962524414, |
| "logps/rejected": -197.67295837402344, |
| "loss": 0.1849, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.3399019241333008, |
| "rewards/margins": 4.401653289794922, |
| "rewards/rejected": -3.061751127243042, |
| "step": 3810 |
| }, |
| { |
| "epoch": 1.39, |
| "grad_norm": 35.29839166964984, |
| "learning_rate": 3.903263423347678e-08, |
| "logits/chosen": -1.9683891534805298, |
| "logits/rejected": -1.4604440927505493, |
| "logps/chosen": -89.45817565917969, |
| "logps/rejected": -148.0181121826172, |
| "loss": 0.1761, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.6148315072059631, |
| "rewards/margins": 3.239147186279297, |
| "rewards/rejected": -2.624316453933716, |
| "step": 3820 |
| }, |
| { |
| "epoch": 1.39, |
| "grad_norm": 39.48429335160676, |
| "learning_rate": 3.8616424755030845e-08, |
| "logits/chosen": -1.5986571311950684, |
| "logits/rejected": -1.3868935108184814, |
| "logps/chosen": -101.1136474609375, |
| "logps/rejected": -143.46604919433594, |
| "loss": 0.1891, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.8768807649612427, |
| "rewards/margins": 3.3683598041534424, |
| "rewards/rejected": -2.4914793968200684, |
| "step": 3830 |
| }, |
| { |
| "epoch": 1.39, |
| "grad_norm": 49.23729367857185, |
| "learning_rate": 3.820167548702516e-08, |
| "logits/chosen": -1.5271122455596924, |
| "logits/rejected": -1.0103445053100586, |
| "logps/chosen": -68.83717346191406, |
| "logps/rejected": -165.7958984375, |
| "loss": 0.1597, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0218682289123535, |
| "rewards/margins": 5.000029563903809, |
| "rewards/rejected": -3.978161334991455, |
| "step": 3840 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 24.868995408783775, |
| "learning_rate": 3.778840307491595e-08, |
| "logits/chosen": -1.700553297996521, |
| "logits/rejected": -1.1538689136505127, |
| "logps/chosen": -91.43379211425781, |
| "logps/rejected": -162.2164306640625, |
| "loss": 0.1497, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1438677310943604, |
| "rewards/margins": 4.128595352172852, |
| "rewards/rejected": -2.9847278594970703, |
| "step": 3850 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 27.603218968201812, |
| "learning_rate": 3.737662410488772e-08, |
| "logits/chosen": -1.768294334411621, |
| "logits/rejected": -1.8469886779785156, |
| "logps/chosen": -95.86221313476562, |
| "logps/rejected": -161.03114318847656, |
| "loss": 0.1792, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.108917236328125, |
| "rewards/margins": 4.2280168533325195, |
| "rewards/rejected": -3.1190993785858154, |
| "step": 3860 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 36.9707293925801, |
| "learning_rate": 3.696635510318747e-08, |
| "logits/chosen": -2.072791814804077, |
| "logits/rejected": -1.609438180923462, |
| "logps/chosen": -96.32298278808594, |
| "logps/rejected": -139.48504638671875, |
| "loss": 0.1769, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.095580816268921, |
| "rewards/margins": 2.998603105545044, |
| "rewards/rejected": -1.9030221700668335, |
| "step": 3870 |
| }, |
| { |
| "epoch": 1.41, |
| "grad_norm": 18.130530465583483, |
| "learning_rate": 3.655761253546142e-08, |
| "logits/chosen": -1.739894151687622, |
| "logits/rejected": -1.5710958242416382, |
| "logps/chosen": -73.37211608886719, |
| "logps/rejected": -147.71841430664062, |
| "loss": 0.1407, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.7478740215301514, |
| "rewards/margins": 4.458775520324707, |
| "rewards/rejected": -2.7109017372131348, |
| "step": 3880 |
| }, |
| { |
| "epoch": 1.41, |
| "grad_norm": 41.20250396110067, |
| "learning_rate": 3.6150412806094344e-08, |
| "logits/chosen": -2.221762180328369, |
| "logits/rejected": -1.7284727096557617, |
| "logps/chosen": -87.04502868652344, |
| "logps/rejected": -178.15328979492188, |
| "loss": 0.1511, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.5952017307281494, |
| "rewards/margins": 4.024810791015625, |
| "rewards/rejected": -3.4296088218688965, |
| "step": 3890 |
| }, |
| { |
| "epoch": 1.42, |
| "grad_norm": 30.898435410912892, |
| "learning_rate": 3.574477225755092e-08, |
| "logits/chosen": -1.9189503192901611, |
| "logits/rejected": -1.6017463207244873, |
| "logps/chosen": -91.14994049072266, |
| "logps/rejected": -140.59140014648438, |
| "loss": 0.1603, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.5078619718551636, |
| "rewards/margins": 3.65142560005188, |
| "rewards/rejected": -2.143563747406006, |
| "step": 3900 |
| }, |
| { |
| "epoch": 1.42, |
| "eval_logits/chosen": -2.447599411010742, |
| "eval_logits/rejected": -2.271988868713379, |
| "eval_logps/chosen": -89.99532318115234, |
| "eval_logps/rejected": -152.36782836914062, |
| "eval_loss": 0.2630765736103058, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": 0.9960936903953552, |
| "eval_rewards/margins": 3.695732831954956, |
| "eval_rewards/rejected": -2.699639081954956, |
| "eval_runtime": 71.5499, |
| "eval_samples_per_second": 12.439, |
| "eval_steps_per_second": 0.196, |
| "step": 3900 |
| }, |
| { |
| "epoch": 1.42, |
| "grad_norm": 15.623201016392487, |
| "learning_rate": 3.534070716972011e-08, |
| "logits/chosen": -1.98562490940094, |
| "logits/rejected": -1.7183992862701416, |
| "logps/chosen": -67.69291687011719, |
| "logps/rejected": -132.70785522460938, |
| "loss": 0.1543, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.898485541343689, |
| "rewards/margins": 4.570878028869629, |
| "rewards/rejected": -2.6723921298980713, |
| "step": 3910 |
| }, |
| { |
| "epoch": 1.42, |
| "grad_norm": 29.502016145046106, |
| "learning_rate": 3.493823375926165e-08, |
| "logits/chosen": -2.040684938430786, |
| "logits/rejected": -1.617761254310608, |
| "logps/chosen": -94.72434997558594, |
| "logps/rejected": -180.46612548828125, |
| "loss": 0.1709, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4578778743743896, |
| "rewards/margins": 4.434561729431152, |
| "rewards/rejected": -2.9766833782196045, |
| "step": 3920 |
| }, |
| { |
| "epoch": 1.43, |
| "grad_norm": 43.61044884590936, |
| "learning_rate": 3.4537368178955237e-08, |
| "logits/chosen": -2.5114989280700684, |
| "logits/rejected": -1.929688811302185, |
| "logps/chosen": -63.439727783203125, |
| "logps/rejected": -167.50003051757812, |
| "loss": 0.1306, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4715431928634644, |
| "rewards/margins": 5.039919853210449, |
| "rewards/rejected": -3.5683765411376953, |
| "step": 3930 |
| }, |
| { |
| "epoch": 1.43, |
| "grad_norm": 31.861912930744854, |
| "learning_rate": 3.4138126517052315e-08, |
| "logits/chosen": -2.129574775695801, |
| "logits/rejected": -2.0275707244873047, |
| "logps/chosen": -91.8274917602539, |
| "logps/rejected": -152.502685546875, |
| "loss": 0.1992, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.9415672421455383, |
| "rewards/margins": 3.8349013328552246, |
| "rewards/rejected": -2.893334150314331, |
| "step": 3940 |
| }, |
| { |
| "epoch": 1.43, |
| "grad_norm": 10.50472857948733, |
| "learning_rate": 3.374052479663024e-08, |
| "logits/chosen": -1.59491765499115, |
| "logits/rejected": -1.3649214506149292, |
| "logps/chosen": -104.72843933105469, |
| "logps/rejected": -193.97207641601562, |
| "loss": 0.1731, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1260311603546143, |
| "rewards/margins": 5.22696590423584, |
| "rewards/rejected": -4.1009345054626465, |
| "step": 3950 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 23.4008383750431, |
| "learning_rate": 3.334457897494941e-08, |
| "logits/chosen": -1.9443241357803345, |
| "logits/rejected": -1.6959731578826904, |
| "logps/chosen": -85.87373352050781, |
| "logps/rejected": -158.98065185546875, |
| "loss": 0.1546, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3743432760238647, |
| "rewards/margins": 4.179642677307129, |
| "rewards/rejected": -2.8052992820739746, |
| "step": 3960 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 33.938523290472, |
| "learning_rate": 3.29503049428127e-08, |
| "logits/chosen": -1.9471511840820312, |
| "logits/rejected": -1.7361555099487305, |
| "logps/chosen": -84.15934753417969, |
| "logps/rejected": -153.57553100585938, |
| "loss": 0.1711, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.410812497138977, |
| "rewards/margins": 4.902033805847168, |
| "rewards/rejected": -3.4912219047546387, |
| "step": 3970 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 42.88422707114712, |
| "learning_rate": 3.255771852392775e-08, |
| "logits/chosen": -1.8716990947723389, |
| "logits/rejected": -1.7490049600601196, |
| "logps/chosen": -93.93757629394531, |
| "logps/rejected": -118.72102355957031, |
| "loss": 0.1712, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.7341419458389282, |
| "rewards/margins": 2.5488927364349365, |
| "rewards/rejected": -1.8147506713867188, |
| "step": 3980 |
| }, |
| { |
| "epoch": 1.45, |
| "grad_norm": 31.96232509403753, |
| "learning_rate": 3.2166835474271995e-08, |
| "logits/chosen": -2.2499547004699707, |
| "logits/rejected": -1.7057373523712158, |
| "logps/chosen": -70.68228912353516, |
| "logps/rejected": -136.07669067382812, |
| "loss": 0.1381, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.8701726794242859, |
| "rewards/margins": 2.541351556777954, |
| "rewards/rejected": -1.671179175376892, |
| "step": 3990 |
| }, |
| { |
| "epoch": 1.45, |
| "grad_norm": 33.013579048970634, |
| "learning_rate": 3.177767148146004e-08, |
| "logits/chosen": -1.196561336517334, |
| "logits/rejected": -0.9995512962341309, |
| "logps/chosen": -89.14258575439453, |
| "logps/rejected": -131.5675048828125, |
| "loss": 0.2054, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6014581918716431, |
| "rewards/margins": 2.8417820930480957, |
| "rewards/rejected": -2.240324020385742, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.45, |
| "eval_logits/chosen": -2.5068588256835938, |
| "eval_logits/rejected": -2.316378116607666, |
| "eval_logps/chosen": -92.20320129394531, |
| "eval_logps/rejected": -153.47879028320312, |
| "eval_loss": 0.25913339853286743, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7753064036369324, |
| "eval_rewards/margins": 3.5860416889190674, |
| "eval_rewards/rejected": -2.8107354640960693, |
| "eval_runtime": 71.5791, |
| "eval_samples_per_second": 12.434, |
| "eval_steps_per_second": 0.196, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.46, |
| "grad_norm": 29.18650004448433, |
| "learning_rate": 3.139024216411438e-08, |
| "logits/chosen": -1.9775199890136719, |
| "logits/rejected": -1.7694860696792603, |
| "logps/chosen": -74.32111358642578, |
| "logps/rejected": -114.33467102050781, |
| "loss": 0.137, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.9357797503471375, |
| "rewards/margins": 3.2922019958496094, |
| "rewards/rejected": -2.356421947479248, |
| "step": 4010 |
| }, |
| { |
| "epoch": 1.46, |
| "grad_norm": 29.240217688028995, |
| "learning_rate": 3.100456307123838e-08, |
| "logits/chosen": -2.0903449058532715, |
| "logits/rejected": -1.7641003131866455, |
| "logps/chosen": -81.42185974121094, |
| "logps/rejected": -165.4561309814453, |
| "loss": 0.1658, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1881463527679443, |
| "rewards/margins": 4.836016654968262, |
| "rewards/rejected": -3.6478705406188965, |
| "step": 4020 |
| }, |
| { |
| "epoch": 1.46, |
| "grad_norm": 30.89971434639723, |
| "learning_rate": 3.062064968159231e-08, |
| "logits/chosen": -2.3161113262176514, |
| "logits/rejected": -1.780773401260376, |
| "logps/chosen": -84.45279693603516, |
| "logps/rejected": -165.2125701904297, |
| "loss": 0.1622, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.193415880203247, |
| "rewards/margins": 4.027709484100342, |
| "rewards/rejected": -2.834293842315674, |
| "step": 4030 |
| }, |
| { |
| "epoch": 1.47, |
| "grad_norm": 17.28427871490954, |
| "learning_rate": 3.023851740307201e-08, |
| "logits/chosen": -2.0731730461120605, |
| "logits/rejected": -1.6455036401748657, |
| "logps/chosen": -110.5876693725586, |
| "logps/rejected": -180.8351287841797, |
| "loss": 0.1633, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.24699239432811737, |
| "rewards/margins": 3.7656092643737793, |
| "rewards/rejected": -3.5186171531677246, |
| "step": 4040 |
| }, |
| { |
| "epoch": 1.47, |
| "grad_norm": 25.73309666117762, |
| "learning_rate": 2.9858181572090675e-08, |
| "logits/chosen": -1.7583844661712646, |
| "logits/rejected": -1.8522754907608032, |
| "logps/chosen": -94.708251953125, |
| "logps/rejected": -162.52867126464844, |
| "loss": 0.1325, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0252021551132202, |
| "rewards/margins": 3.6236395835876465, |
| "rewards/rejected": -2.5984373092651367, |
| "step": 4050 |
| }, |
| { |
| "epoch": 1.47, |
| "grad_norm": 31.111251255931403, |
| "learning_rate": 2.9479657452963256e-08, |
| "logits/chosen": -2.248737096786499, |
| "logits/rejected": -1.7086880207061768, |
| "logps/chosen": -104.32584381103516, |
| "logps/rejected": -170.23004150390625, |
| "loss": 0.151, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.08391883969306946, |
| "rewards/margins": 3.075791835784912, |
| "rewards/rejected": -2.991872787475586, |
| "step": 4060 |
| }, |
| { |
| "epoch": 1.48, |
| "grad_norm": 18.513737492370375, |
| "learning_rate": 2.910296023729384e-08, |
| "logits/chosen": -1.4177262783050537, |
| "logits/rejected": -1.364580750465393, |
| "logps/chosen": -89.30236053466797, |
| "logps/rejected": -147.00576782226562, |
| "loss": 0.1544, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0421960353851318, |
| "rewards/margins": 3.4464340209960938, |
| "rewards/rejected": -2.404238224029541, |
| "step": 4070 |
| }, |
| { |
| "epoch": 1.48, |
| "grad_norm": 45.87936243907442, |
| "learning_rate": 2.8728105043365984e-08, |
| "logits/chosen": -1.8459665775299072, |
| "logits/rejected": -1.5093791484832764, |
| "logps/chosen": -97.5422592163086, |
| "logps/rejected": -131.14620971679688, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.9450712203979492, |
| "rewards/margins": 3.339611530303955, |
| "rewards/rejected": -2.394540309906006, |
| "step": 4080 |
| }, |
| { |
| "epoch": 1.48, |
| "grad_norm": 27.364987131956532, |
| "learning_rate": 2.8355106915535932e-08, |
| "logits/chosen": -1.9753601551055908, |
| "logits/rejected": -1.554747462272644, |
| "logps/chosen": -72.57929229736328, |
| "logps/rejected": -136.12213134765625, |
| "loss": 0.135, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8972055315971375, |
| "rewards/margins": 4.053958415985107, |
| "rewards/rejected": -3.156752824783325, |
| "step": 4090 |
| }, |
| { |
| "epoch": 1.49, |
| "grad_norm": 32.389832751742276, |
| "learning_rate": 2.798398082362886e-08, |
| "logits/chosen": -1.6929162740707397, |
| "logits/rejected": -1.7215791940689087, |
| "logps/chosen": -115.38690185546875, |
| "logps/rejected": -146.203125, |
| "loss": 0.1413, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.22363097965717316, |
| "rewards/margins": 3.6949760913848877, |
| "rewards/rejected": -3.4713454246520996, |
| "step": 4100 |
| }, |
| { |
| "epoch": 1.49, |
| "eval_logits/chosen": -2.485478639602661, |
| "eval_logits/rejected": -2.2992606163024902, |
| "eval_logps/chosen": -91.98760223388672, |
| "eval_logps/rejected": -154.0671844482422, |
| "eval_loss": 0.26037731766700745, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.796866238117218, |
| "eval_rewards/margins": 3.6664421558380127, |
| "eval_rewards/rejected": -2.8695759773254395, |
| "eval_runtime": 71.6081, |
| "eval_samples_per_second": 12.429, |
| "eval_steps_per_second": 0.196, |
| "step": 4100 |
| }, |
| { |
| "epoch": 1.49, |
| "grad_norm": 14.2142969265953, |
| "learning_rate": 2.761474166233805e-08, |
| "logits/chosen": -1.9014062881469727, |
| "logits/rejected": -1.554809331893921, |
| "logps/chosen": -91.71360778808594, |
| "logps/rejected": -198.91380310058594, |
| "loss": 0.1107, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8375331163406372, |
| "rewards/margins": 5.089506149291992, |
| "rewards/rejected": -4.2519731521606445, |
| "step": 4110 |
| }, |
| { |
| "epoch": 1.5, |
| "grad_norm": 36.46497820122211, |
| "learning_rate": 2.724740425062714e-08, |
| "logits/chosen": -1.5665128231048584, |
| "logits/rejected": -1.6638376712799072, |
| "logps/chosen": -97.43113708496094, |
| "logps/rejected": -185.1346893310547, |
| "loss": 0.1309, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.911544680595398, |
| "rewards/margins": 4.338427543640137, |
| "rewards/rejected": -3.4268829822540283, |
| "step": 4120 |
| }, |
| { |
| "epoch": 1.5, |
| "grad_norm": 50.17754349555712, |
| "learning_rate": 2.6881983331135378e-08, |
| "logits/chosen": -1.887635588645935, |
| "logits/rejected": -1.7573896646499634, |
| "logps/chosen": -80.06953430175781, |
| "logps/rejected": -133.02284240722656, |
| "loss": 0.1722, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.585761547088623, |
| "rewards/margins": 3.1335628032684326, |
| "rewards/rejected": -2.5478012561798096, |
| "step": 4130 |
| }, |
| { |
| "epoch": 1.5, |
| "grad_norm": 22.44755921748364, |
| "learning_rate": 2.6518493569585857e-08, |
| "logits/chosen": -1.984948754310608, |
| "logits/rejected": -2.0342633724212646, |
| "logps/chosen": -114.79627990722656, |
| "logps/rejected": -142.2074432373047, |
| "loss": 0.199, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.6468585729598999, |
| "rewards/margins": 2.1931614875793457, |
| "rewards/rejected": -1.546303153038025, |
| "step": 4140 |
| }, |
| { |
| "epoch": 1.51, |
| "grad_norm": 24.955293029439613, |
| "learning_rate": 2.6156949554197095e-08, |
| "logits/chosen": -2.307645797729492, |
| "logits/rejected": -1.599973201751709, |
| "logps/chosen": -90.35932922363281, |
| "logps/rejected": -148.82672119140625, |
| "loss": 0.1611, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.5787138938903809, |
| "rewards/margins": 3.0710272789001465, |
| "rewards/rejected": -2.492313861846924, |
| "step": 4150 |
| }, |
| { |
| "epoch": 1.51, |
| "grad_norm": 36.71467993455758, |
| "learning_rate": 2.5797365795097407e-08, |
| "logits/chosen": -1.9828819036483765, |
| "logits/rejected": -1.6299062967300415, |
| "logps/chosen": -103.05985260009766, |
| "logps/rejected": -187.84640502929688, |
| "loss": 0.1561, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7202814817428589, |
| "rewards/margins": 4.248810768127441, |
| "rewards/rejected": -3.528529405593872, |
| "step": 4160 |
| }, |
| { |
| "epoch": 1.51, |
| "grad_norm": 25.40867584669409, |
| "learning_rate": 2.543975672374264e-08, |
| "logits/chosen": -1.7713234424591064, |
| "logits/rejected": -1.78484308719635, |
| "logps/chosen": -91.5001220703125, |
| "logps/rejected": -149.6800079345703, |
| "loss": 0.1647, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.5808213353157043, |
| "rewards/margins": 3.1474668979644775, |
| "rewards/rejected": -2.566645383834839, |
| "step": 4170 |
| }, |
| { |
| "epoch": 1.52, |
| "grad_norm": 19.8252963506834, |
| "learning_rate": 2.5084136692336926e-08, |
| "logits/chosen": -1.3749628067016602, |
| "logits/rejected": -1.4281375408172607, |
| "logps/chosen": -70.6727066040039, |
| "logps/rejected": -277.53790283203125, |
| "loss": 0.1486, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1151177883148193, |
| "rewards/margins": 4.473038196563721, |
| "rewards/rejected": -3.3579201698303223, |
| "step": 4180 |
| }, |
| { |
| "epoch": 1.52, |
| "grad_norm": 22.788082275545953, |
| "learning_rate": 2.4730519973256725e-08, |
| "logits/chosen": -1.382116436958313, |
| "logits/rejected": -1.2883073091506958, |
| "logps/chosen": -77.42723083496094, |
| "logps/rejected": -168.14895629882812, |
| "loss": 0.1365, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.9808204770088196, |
| "rewards/margins": 4.480748176574707, |
| "rewards/rejected": -3.499927520751953, |
| "step": 4190 |
| }, |
| { |
| "epoch": 1.52, |
| "grad_norm": 17.55083100932304, |
| "learning_rate": 2.4378920758477996e-08, |
| "logits/chosen": -2.1403841972351074, |
| "logits/rejected": -1.4080195426940918, |
| "logps/chosen": -69.2994384765625, |
| "logps/rejected": -133.26760864257812, |
| "loss": 0.1498, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.458009958267212, |
| "rewards/margins": 3.704465389251709, |
| "rewards/rejected": -2.246455669403076, |
| "step": 4200 |
| }, |
| { |
| "epoch": 1.52, |
| "eval_logits/chosen": -2.486311435699463, |
| "eval_logits/rejected": -2.3020877838134766, |
| "eval_logps/chosen": -91.95867919921875, |
| "eval_logps/rejected": -154.05789184570312, |
| "eval_loss": 0.2579483687877655, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7997574210166931, |
| "eval_rewards/margins": 3.6684017181396484, |
| "eval_rewards/rejected": -2.8686444759368896, |
| "eval_runtime": 71.505, |
| "eval_samples_per_second": 12.447, |
| "eval_steps_per_second": 0.196, |
| "step": 4200 |
| }, |
| { |
| "epoch": 1.53, |
| "grad_norm": 35.97397604275955, |
| "learning_rate": 2.4029353159006606e-08, |
| "logits/chosen": -1.889622688293457, |
| "logits/rejected": -1.7178815603256226, |
| "logps/chosen": -70.93376159667969, |
| "logps/rejected": -144.45614624023438, |
| "loss": 0.1486, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.563561201095581, |
| "rewards/margins": 4.030331611633301, |
| "rewards/rejected": -2.466770648956299, |
| "step": 4210 |
| }, |
| { |
| "epoch": 1.53, |
| "grad_norm": 25.472759515203542, |
| "learning_rate": 2.368183120431205e-08, |
| "logits/chosen": -1.6090589761734009, |
| "logits/rejected": -1.5134804248809814, |
| "logps/chosen": -93.47647857666016, |
| "logps/rejected": -154.41531372070312, |
| "loss": 0.1534, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2347394227981567, |
| "rewards/margins": 4.24044942855835, |
| "rewards/rejected": -3.0057103633880615, |
| "step": 4220 |
| }, |
| { |
| "epoch": 1.54, |
| "grad_norm": 26.748913628743217, |
| "learning_rate": 2.3336368841764356e-08, |
| "logits/chosen": -1.6902498006820679, |
| "logits/rejected": -1.2772510051727295, |
| "logps/chosen": -72.32806396484375, |
| "logps/rejected": -127.91917419433594, |
| "loss": 0.105, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5376206636428833, |
| "rewards/margins": 3.2143516540527344, |
| "rewards/rejected": -1.676730751991272, |
| "step": 4230 |
| }, |
| { |
| "epoch": 1.54, |
| "grad_norm": 38.70782633546678, |
| "learning_rate": 2.2992979936074264e-08, |
| "logits/chosen": -2.003537654876709, |
| "logits/rejected": -1.539896011352539, |
| "logps/chosen": -107.56465148925781, |
| "logps/rejected": -155.28176879882812, |
| "loss": 0.1745, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.1425919532775879, |
| "rewards/margins": 2.318450927734375, |
| "rewards/rejected": -2.461042881011963, |
| "step": 4240 |
| }, |
| { |
| "epoch": 1.54, |
| "grad_norm": 29.839188839127143, |
| "learning_rate": 2.2651678268736942e-08, |
| "logits/chosen": -1.5676755905151367, |
| "logits/rejected": -1.5963351726531982, |
| "logps/chosen": -117.62342834472656, |
| "logps/rejected": -148.70346069335938, |
| "loss": 0.1028, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9985088109970093, |
| "rewards/margins": 3.8390114307403564, |
| "rewards/rejected": -2.840503215789795, |
| "step": 4250 |
| }, |
| { |
| "epoch": 1.55, |
| "grad_norm": 19.326703363193936, |
| "learning_rate": 2.2312477537478763e-08, |
| "logits/chosen": -2.033979892730713, |
| "logits/rejected": -1.5905263423919678, |
| "logps/chosen": -78.28575134277344, |
| "logps/rejected": -167.78639221191406, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2223570346832275, |
| "rewards/margins": 4.399596214294434, |
| "rewards/rejected": -3.1772396564483643, |
| "step": 4260 |
| }, |
| { |
| "epoch": 1.55, |
| "grad_norm": 34.56110655069492, |
| "learning_rate": 2.1975391355707567e-08, |
| "logits/chosen": -2.090172529220581, |
| "logits/rejected": -1.7063558101654053, |
| "logps/chosen": -69.6578369140625, |
| "logps/rejected": -146.59194946289062, |
| "loss": 0.1482, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4781410694122314, |
| "rewards/margins": 4.314321994781494, |
| "rewards/rejected": -2.836181163787842, |
| "step": 4270 |
| }, |
| { |
| "epoch": 1.55, |
| "grad_norm": 37.15442072437241, |
| "learning_rate": 2.164043325196635e-08, |
| "logits/chosen": -1.6971238851547241, |
| "logits/rejected": -1.8592636585235596, |
| "logps/chosen": -82.22840881347656, |
| "logps/rejected": -163.99058532714844, |
| "loss": 0.134, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3080239295959473, |
| "rewards/margins": 3.911048412322998, |
| "rewards/rejected": -2.60302472114563, |
| "step": 4280 |
| }, |
| { |
| "epoch": 1.56, |
| "grad_norm": 22.652340412985147, |
| "learning_rate": 2.13076166693903e-08, |
| "logits/chosen": -2.017364501953125, |
| "logits/rejected": -1.9596898555755615, |
| "logps/chosen": -100.14933776855469, |
| "logps/rejected": -186.43014526367188, |
| "loss": 0.1298, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3219308853149414, |
| "rewards/margins": 4.975711822509766, |
| "rewards/rejected": -3.653780698776245, |
| "step": 4290 |
| }, |
| { |
| "epoch": 1.56, |
| "grad_norm": 26.988430640272753, |
| "learning_rate": 2.0976954965167228e-08, |
| "logits/chosen": -2.23913311958313, |
| "logits/rejected": -2.0381526947021484, |
| "logps/chosen": -100.4576644897461, |
| "logps/rejected": -205.36434936523438, |
| "loss": 0.174, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9682433009147644, |
| "rewards/margins": 4.4989728927612305, |
| "rewards/rejected": -3.5307304859161377, |
| "step": 4300 |
| }, |
| { |
| "epoch": 1.56, |
| "eval_logits/chosen": -2.39996600151062, |
| "eval_logits/rejected": -2.239955186843872, |
| "eval_logps/chosen": -92.15650939941406, |
| "eval_logps/rejected": -155.64747619628906, |
| "eval_loss": 0.26092293858528137, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7799752354621887, |
| "eval_rewards/margins": 3.8075778484344482, |
| "eval_rewards/rejected": -3.0276029109954834, |
| "eval_runtime": 71.3862, |
| "eval_samples_per_second": 12.467, |
| "eval_steps_per_second": 0.196, |
| "step": 4300 |
| }, |
| { |
| "epoch": 1.56, |
| "grad_norm": 20.13961405134126, |
| "learning_rate": 2.064846141000156e-08, |
| "logits/chosen": -1.785154104232788, |
| "logits/rejected": -1.782621145248413, |
| "logps/chosen": -100.51982116699219, |
| "logps/rejected": -170.2032012939453, |
| "loss": 0.1628, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1105843782424927, |
| "rewards/margins": 4.3706769943237305, |
| "rewards/rejected": -3.260092258453369, |
| "step": 4310 |
| }, |
| { |
| "epoch": 1.57, |
| "grad_norm": 26.57133171694419, |
| "learning_rate": 2.0322149187581696e-08, |
| "logits/chosen": -2.1262426376342773, |
| "logits/rejected": -1.7420053482055664, |
| "logps/chosen": -85.6638412475586, |
| "logps/rejected": -192.35430908203125, |
| "loss": 0.1245, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9371218681335449, |
| "rewards/margins": 5.653896331787109, |
| "rewards/rejected": -4.7167744636535645, |
| "step": 4320 |
| }, |
| { |
| "epoch": 1.57, |
| "grad_norm": 30.486425514760217, |
| "learning_rate": 1.9998031394050925e-08, |
| "logits/chosen": -2.0862393379211426, |
| "logits/rejected": -1.73501718044281, |
| "logps/chosen": -110.37203216552734, |
| "logps/rejected": -184.22003173828125, |
| "loss": 0.1412, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.701205849647522, |
| "rewards/margins": 4.286178112030029, |
| "rewards/rejected": -3.584972381591797, |
| "step": 4330 |
| }, |
| { |
| "epoch": 1.58, |
| "grad_norm": 29.613748447060853, |
| "learning_rate": 1.9676121037481733e-08, |
| "logits/chosen": -1.8443056344985962, |
| "logits/rejected": -1.37973153591156, |
| "logps/chosen": -87.62504577636719, |
| "logps/rejected": -148.2349090576172, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0517264604568481, |
| "rewards/margins": 3.312018632888794, |
| "rewards/rejected": -2.2602920532226562, |
| "step": 4340 |
| }, |
| { |
| "epoch": 1.58, |
| "grad_norm": 38.941503925465945, |
| "learning_rate": 1.935643103735389e-08, |
| "logits/chosen": -2.3647444248199463, |
| "logits/rejected": -1.6877624988555908, |
| "logps/chosen": -78.06304168701172, |
| "logps/rejected": -170.48617553710938, |
| "loss": 0.1479, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1019824743270874, |
| "rewards/margins": 4.5098443031311035, |
| "rewards/rejected": -3.4078621864318848, |
| "step": 4350 |
| }, |
| { |
| "epoch": 1.58, |
| "grad_norm": 50.92032810215081, |
| "learning_rate": 1.9038974224035848e-08, |
| "logits/chosen": -1.945586919784546, |
| "logits/rejected": -1.535428762435913, |
| "logps/chosen": -93.87750244140625, |
| "logps/rejected": -185.80007934570312, |
| "loss": 0.1745, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.7316128611564636, |
| "rewards/margins": 3.479790210723877, |
| "rewards/rejected": -2.7481772899627686, |
| "step": 4360 |
| }, |
| { |
| "epoch": 1.59, |
| "grad_norm": 33.56498772715042, |
| "learning_rate": 1.8723763338269824e-08, |
| "logits/chosen": -2.1573054790496826, |
| "logits/rejected": -1.6914573907852173, |
| "logps/chosen": -97.86043548583984, |
| "logps/rejected": -163.97647094726562, |
| "loss": 0.1815, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.35831624269485474, |
| "rewards/margins": 3.7290472984313965, |
| "rewards/rejected": -3.3707308769226074, |
| "step": 4370 |
| }, |
| { |
| "epoch": 1.59, |
| "grad_norm": 26.65376761598817, |
| "learning_rate": 1.8410811030660466e-08, |
| "logits/chosen": -1.8529369831085205, |
| "logits/rejected": -1.8225460052490234, |
| "logps/chosen": -126.21983337402344, |
| "logps/rejected": -171.356689453125, |
| "loss": 0.1578, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.27563539147377014, |
| "rewards/margins": 3.433450222015381, |
| "rewards/rejected": -3.1578147411346436, |
| "step": 4380 |
| }, |
| { |
| "epoch": 1.59, |
| "grad_norm": 41.658972724370116, |
| "learning_rate": 1.810012986116715e-08, |
| "logits/chosen": -2.1115715503692627, |
| "logits/rejected": -1.5005934238433838, |
| "logps/chosen": -78.10107421875, |
| "logps/rejected": -153.611572265625, |
| "loss": 0.1355, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1123895645141602, |
| "rewards/margins": 4.308642387390137, |
| "rewards/rejected": -3.1962530612945557, |
| "step": 4390 |
| }, |
| { |
| "epoch": 1.6, |
| "grad_norm": 23.104818947703635, |
| "learning_rate": 1.7791732298599888e-08, |
| "logits/chosen": -1.8663402795791626, |
| "logits/rejected": -1.7844841480255127, |
| "logps/chosen": -83.04045104980469, |
| "logps/rejected": -153.29029846191406, |
| "loss": 0.1129, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5516669750213623, |
| "rewards/margins": 4.48703670501709, |
| "rewards/rejected": -2.9353702068328857, |
| "step": 4400 |
| }, |
| { |
| "epoch": 1.6, |
| "eval_logits/chosen": -2.4757511615753174, |
| "eval_logits/rejected": -2.2932116985321045, |
| "eval_logps/chosen": -92.66739654541016, |
| "eval_logps/rejected": -155.24755859375, |
| "eval_loss": 0.2576039731502533, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7288867235183716, |
| "eval_rewards/margins": 3.716498851776123, |
| "eval_rewards/rejected": -2.987612009048462, |
| "eval_runtime": 71.4688, |
| "eval_samples_per_second": 12.453, |
| "eval_steps_per_second": 0.196, |
| "step": 4400 |
| }, |
| { |
| "epoch": 1.6, |
| "grad_norm": 30.602138050700304, |
| "learning_rate": 1.7485630720118904e-08, |
| "logits/chosen": -1.7544384002685547, |
| "logits/rejected": -1.5731135606765747, |
| "logps/chosen": -95.06303405761719, |
| "logps/rejected": -155.60504150390625, |
| "loss": 0.1394, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6706847548484802, |
| "rewards/margins": 3.354102373123169, |
| "rewards/rejected": -2.683417320251465, |
| "step": 4410 |
| }, |
| { |
| "epoch": 1.6, |
| "grad_norm": 24.591287815914352, |
| "learning_rate": 1.7181837410737932e-08, |
| "logits/chosen": -1.6455243825912476, |
| "logits/rejected": -1.8114473819732666, |
| "logps/chosen": -91.33721923828125, |
| "logps/rejected": -158.2550811767578, |
| "loss": 0.1183, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.6706674098968506, |
| "rewards/margins": 3.6940701007843018, |
| "rewards/rejected": -2.023402690887451, |
| "step": 4420 |
| }, |
| { |
| "epoch": 1.61, |
| "grad_norm": 28.285812502384303, |
| "learning_rate": 1.688036456283108e-08, |
| "logits/chosen": -1.4960906505584717, |
| "logits/rejected": -1.562246322631836, |
| "logps/chosen": -92.74504089355469, |
| "logps/rejected": -129.95484924316406, |
| "loss": 0.1344, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7566136121749878, |
| "rewards/margins": 3.488935947418213, |
| "rewards/rejected": -2.7323222160339355, |
| "step": 4430 |
| }, |
| { |
| "epoch": 1.61, |
| "grad_norm": 21.99612078068548, |
| "learning_rate": 1.6581224275643602e-08, |
| "logits/chosen": -2.4308037757873535, |
| "logits/rejected": -2.031317710876465, |
| "logps/chosen": -81.02774810791016, |
| "logps/rejected": -149.41329956054688, |
| "loss": 0.124, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0930092334747314, |
| "rewards/margins": 3.704552173614502, |
| "rewards/rejected": -2.6115427017211914, |
| "step": 4440 |
| }, |
| { |
| "epoch": 1.62, |
| "grad_norm": 37.76234741898126, |
| "learning_rate": 1.6284428554806282e-08, |
| "logits/chosen": -1.8159290552139282, |
| "logits/rejected": -1.7262006998062134, |
| "logps/chosen": -84.27841186523438, |
| "logps/rejected": -133.88829040527344, |
| "loss": 0.1195, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7679897546768188, |
| "rewards/margins": 3.5734684467315674, |
| "rewards/rejected": -2.805478572845459, |
| "step": 4450 |
| }, |
| { |
| "epoch": 1.62, |
| "grad_norm": 20.751714992592504, |
| "learning_rate": 1.598998931185358e-08, |
| "logits/chosen": -1.6911875009536743, |
| "logits/rejected": -1.4998903274536133, |
| "logps/chosen": -85.60092163085938, |
| "logps/rejected": -157.67355346679688, |
| "loss": 0.1644, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.2675038576126099, |
| "rewards/margins": 3.786600112915039, |
| "rewards/rejected": -2.5190961360931396, |
| "step": 4460 |
| }, |
| { |
| "epoch": 1.62, |
| "grad_norm": 31.961939419295213, |
| "learning_rate": 1.5697918363745567e-08, |
| "logits/chosen": -1.8280389308929443, |
| "logits/rejected": -1.5867749452590942, |
| "logps/chosen": -108.87247467041016, |
| "logps/rejected": -214.6339111328125, |
| "loss": 0.1564, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.9745191335678101, |
| "rewards/margins": 5.607847213745117, |
| "rewards/rejected": -4.633328437805176, |
| "step": 4470 |
| }, |
| { |
| "epoch": 1.63, |
| "grad_norm": 38.17110938660031, |
| "learning_rate": 1.5408227432393714e-08, |
| "logits/chosen": -2.4660305976867676, |
| "logits/rejected": -1.5764108896255493, |
| "logps/chosen": -89.4808120727539, |
| "logps/rejected": -156.589111328125, |
| "loss": 0.1159, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.1624079942703247, |
| "rewards/margins": 3.693568706512451, |
| "rewards/rejected": -2.531160593032837, |
| "step": 4480 |
| }, |
| { |
| "epoch": 1.63, |
| "grad_norm": 36.10544360692532, |
| "learning_rate": 1.5120928144190412e-08, |
| "logits/chosen": -1.9956271648406982, |
| "logits/rejected": -1.7658560276031494, |
| "logps/chosen": -117.41062927246094, |
| "logps/rejected": -145.37120056152344, |
| "loss": 0.1571, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.006312882993370295, |
| "rewards/margins": 2.9415173530578613, |
| "rewards/rejected": -2.93520450592041, |
| "step": 4490 |
| }, |
| { |
| "epoch": 1.63, |
| "grad_norm": 27.729025187636683, |
| "learning_rate": 1.483603202954238e-08, |
| "logits/chosen": -1.9484901428222656, |
| "logits/rejected": -1.8669170141220093, |
| "logps/chosen": -92.9903335571289, |
| "logps/rejected": -148.825927734375, |
| "loss": 0.1424, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7940378189086914, |
| "rewards/margins": 3.46022367477417, |
| "rewards/rejected": -2.6661858558654785, |
| "step": 4500 |
| }, |
| { |
| "epoch": 1.63, |
| "eval_logits/chosen": -2.4859211444854736, |
| "eval_logits/rejected": -2.302330732345581, |
| "eval_logps/chosen": -92.06935119628906, |
| "eval_logps/rejected": -154.83358764648438, |
| "eval_loss": 0.2585107982158661, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7886915802955627, |
| "eval_rewards/margins": 3.734905242919922, |
| "eval_rewards/rejected": -2.9462132453918457, |
| "eval_runtime": 71.5035, |
| "eval_samples_per_second": 12.447, |
| "eval_steps_per_second": 0.196, |
| "step": 4500 |
| }, |
| { |
| "epoch": 1.64, |
| "grad_norm": 20.03094242415938, |
| "learning_rate": 1.4553550522407868e-08, |
| "logits/chosen": -1.8428974151611328, |
| "logits/rejected": -1.440744161605835, |
| "logps/chosen": -84.1056137084961, |
| "logps/rejected": -152.34756469726562, |
| "loss": 0.1426, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6048812866210938, |
| "rewards/margins": 4.184465408325195, |
| "rewards/rejected": -2.5795845985412598, |
| "step": 4510 |
| }, |
| { |
| "epoch": 1.64, |
| "grad_norm": 34.20593083265495, |
| "learning_rate": 1.4273494959837854e-08, |
| "logits/chosen": -1.979557752609253, |
| "logits/rejected": -1.8085553646087646, |
| "logps/chosen": -87.48872375488281, |
| "logps/rejected": -144.94422912597656, |
| "loss": 0.1596, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.7250168323516846, |
| "rewards/margins": 3.676483154296875, |
| "rewards/rejected": -1.9514662027359009, |
| "step": 4520 |
| }, |
| { |
| "epoch": 1.64, |
| "grad_norm": 15.481458090321123, |
| "learning_rate": 1.3995876581520893e-08, |
| "logits/chosen": -1.636747121810913, |
| "logits/rejected": -1.7493568658828735, |
| "logps/chosen": -104.45011901855469, |
| "logps/rejected": -178.22219848632812, |
| "loss": 0.1425, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.7096387147903442, |
| "rewards/margins": 4.242130756378174, |
| "rewards/rejected": -3.5324923992156982, |
| "step": 4530 |
| }, |
| { |
| "epoch": 1.65, |
| "grad_norm": 26.973245248163447, |
| "learning_rate": 1.3720706529332202e-08, |
| "logits/chosen": -2.262129545211792, |
| "logits/rejected": -1.6040267944335938, |
| "logps/chosen": -77.4686279296875, |
| "logps/rejected": -172.17333984375, |
| "loss": 0.1379, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.497057318687439, |
| "rewards/margins": 4.911464691162109, |
| "rewards/rejected": -3.414407253265381, |
| "step": 4540 |
| }, |
| { |
| "epoch": 1.65, |
| "grad_norm": 24.948661161132325, |
| "learning_rate": 1.3447995846886393e-08, |
| "logits/chosen": -1.9659942388534546, |
| "logits/rejected": -1.7948821783065796, |
| "logps/chosen": -74.55500793457031, |
| "logps/rejected": -173.8816680908203, |
| "loss": 0.149, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5743433237075806, |
| "rewards/margins": 5.000136375427246, |
| "rewards/rejected": -3.425793409347534, |
| "step": 4550 |
| }, |
| { |
| "epoch": 1.66, |
| "grad_norm": 27.776188946775548, |
| "learning_rate": 1.317775547909426e-08, |
| "logits/chosen": -1.6119670867919922, |
| "logits/rejected": -1.385824203491211, |
| "logps/chosen": -96.07972717285156, |
| "logps/rejected": -175.02867126464844, |
| "loss": 0.1963, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7563056945800781, |
| "rewards/margins": 3.682525634765625, |
| "rewards/rejected": -2.926219940185547, |
| "step": 4560 |
| }, |
| { |
| "epoch": 1.66, |
| "grad_norm": 43.370687625677974, |
| "learning_rate": 1.2909996271723539e-08, |
| "logits/chosen": -2.330936908721924, |
| "logits/rejected": -1.7249195575714111, |
| "logps/chosen": -81.22795104980469, |
| "logps/rejected": -180.45004272460938, |
| "loss": 0.1118, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1179533004760742, |
| "rewards/margins": 4.484683990478516, |
| "rewards/rejected": -3.3667304515838623, |
| "step": 4570 |
| }, |
| { |
| "epoch": 1.66, |
| "grad_norm": 52.192177435639636, |
| "learning_rate": 1.2644728970963616e-08, |
| "logits/chosen": -2.1745333671569824, |
| "logits/rejected": -1.5888748168945312, |
| "logps/chosen": -76.57296752929688, |
| "logps/rejected": -189.40585327148438, |
| "loss": 0.1714, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.506654143333435, |
| "rewards/margins": 5.129171371459961, |
| "rewards/rejected": -3.6225173473358154, |
| "step": 4580 |
| }, |
| { |
| "epoch": 1.67, |
| "grad_norm": 15.461194155069908, |
| "learning_rate": 1.2381964222994248e-08, |
| "logits/chosen": -1.6822162866592407, |
| "logits/rejected": -1.4808707237243652, |
| "logps/chosen": -88.69744110107422, |
| "logps/rejected": -152.0595245361328, |
| "loss": 0.1573, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7915393710136414, |
| "rewards/margins": 4.33894157409668, |
| "rewards/rejected": -3.5474014282226562, |
| "step": 4590 |
| }, |
| { |
| "epoch": 1.67, |
| "grad_norm": 30.84506660543241, |
| "learning_rate": 1.2121712573558262e-08, |
| "logits/chosen": -1.8852580785751343, |
| "logits/rejected": -1.933457374572754, |
| "logps/chosen": -88.162841796875, |
| "logps/rejected": -145.27273559570312, |
| "loss": 0.1531, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.409010410308838, |
| "rewards/margins": 4.220501899719238, |
| "rewards/rejected": -2.8114914894104004, |
| "step": 4600 |
| }, |
| { |
| "epoch": 1.67, |
| "eval_logits/chosen": -2.4970266819000244, |
| "eval_logits/rejected": -2.308969020843506, |
| "eval_logps/chosen": -92.29559326171875, |
| "eval_logps/rejected": -154.68136596679688, |
| "eval_loss": 0.257046639919281, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7660664319992065, |
| "eval_rewards/margins": 3.6970560550689697, |
| "eval_rewards/rejected": -2.930989980697632, |
| "eval_runtime": 71.5124, |
| "eval_samples_per_second": 12.445, |
| "eval_steps_per_second": 0.196, |
| "step": 4600 |
| }, |
| { |
| "epoch": 1.67, |
| "grad_norm": 32.065023239868225, |
| "learning_rate": 1.1863984467538368e-08, |
| "logits/chosen": -1.5617568492889404, |
| "logits/rejected": -1.5484387874603271, |
| "logps/chosen": -84.5037612915039, |
| "logps/rejected": -140.86459350585938, |
| "loss": 0.1778, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0827562808990479, |
| "rewards/margins": 4.16703987121582, |
| "rewards/rejected": -3.0842833518981934, |
| "step": 4610 |
| }, |
| { |
| "epoch": 1.68, |
| "grad_norm": 40.53992535432375, |
| "learning_rate": 1.1608790248537947e-08, |
| "logits/chosen": -1.849543809890747, |
| "logits/rejected": -1.3818111419677734, |
| "logps/chosen": -98.61241912841797, |
| "logps/rejected": -204.64102172851562, |
| "loss": 0.1815, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.39084136486053467, |
| "rewards/margins": 4.119144916534424, |
| "rewards/rejected": -3.7283034324645996, |
| "step": 4620 |
| }, |
| { |
| "epoch": 1.68, |
| "grad_norm": 44.60677557796461, |
| "learning_rate": 1.1356140158465846e-08, |
| "logits/chosen": -1.6492226123809814, |
| "logits/rejected": -1.5845674276351929, |
| "logps/chosen": -99.52052307128906, |
| "logps/rejected": -152.729248046875, |
| "loss": 0.152, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.465355783700943, |
| "rewards/margins": 3.311537981033325, |
| "rewards/rejected": -2.846181631088257, |
| "step": 4630 |
| }, |
| { |
| "epoch": 1.68, |
| "grad_norm": 38.975584748722106, |
| "learning_rate": 1.1106044337125478e-08, |
| "logits/chosen": -1.9782928228378296, |
| "logits/rejected": -1.9134845733642578, |
| "logps/chosen": -93.96483612060547, |
| "logps/rejected": -147.70205688476562, |
| "loss": 0.1498, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.3415434956550598, |
| "rewards/margins": 3.8670287132263184, |
| "rewards/rejected": -3.5254852771759033, |
| "step": 4640 |
| }, |
| { |
| "epoch": 1.69, |
| "grad_norm": 30.98909384785237, |
| "learning_rate": 1.0858512821807742e-08, |
| "logits/chosen": -1.6544148921966553, |
| "logits/rejected": -1.434407114982605, |
| "logps/chosen": -97.20882415771484, |
| "logps/rejected": -143.4949188232422, |
| "loss": 0.184, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.9800602793693542, |
| "rewards/margins": 2.9634454250335693, |
| "rewards/rejected": -1.9833850860595703, |
| "step": 4650 |
| }, |
| { |
| "epoch": 1.69, |
| "grad_norm": 37.51557850755165, |
| "learning_rate": 1.0613555546888275e-08, |
| "logits/chosen": -1.5016518831253052, |
| "logits/rejected": -1.68484628200531, |
| "logps/chosen": -88.76266479492188, |
| "logps/rejected": -165.8483123779297, |
| "loss": 0.1375, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8447409868240356, |
| "rewards/margins": 4.642016887664795, |
| "rewards/rejected": -3.797276020050049, |
| "step": 4660 |
| }, |
| { |
| "epoch": 1.7, |
| "grad_norm": 38.74297206743728, |
| "learning_rate": 1.0371182343428694e-08, |
| "logits/chosen": -1.381477952003479, |
| "logits/rejected": -2.027268171310425, |
| "logps/chosen": -98.62785339355469, |
| "logps/rejected": -132.6935577392578, |
| "loss": 0.1417, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.7803923487663269, |
| "rewards/margins": 3.797145366668701, |
| "rewards/rejected": -3.0167534351348877, |
| "step": 4670 |
| }, |
| { |
| "epoch": 1.7, |
| "grad_norm": 37.59159294477782, |
| "learning_rate": 1.0131402938782063e-08, |
| "logits/chosen": -1.829708456993103, |
| "logits/rejected": -1.5681079626083374, |
| "logps/chosen": -89.08270263671875, |
| "logps/rejected": -157.6560821533203, |
| "loss": 0.1658, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0118838548660278, |
| "rewards/margins": 4.21151065826416, |
| "rewards/rejected": -3.19962739944458, |
| "step": 4680 |
| }, |
| { |
| "epoch": 1.7, |
| "grad_norm": 18.933900384534923, |
| "learning_rate": 9.894226956202484e-09, |
| "logits/chosen": -1.6854585409164429, |
| "logits/rejected": -1.6041696071624756, |
| "logps/chosen": -87.95982360839844, |
| "logps/rejected": -164.80075073242188, |
| "loss": 0.1541, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1225395202636719, |
| "rewards/margins": 4.531619071960449, |
| "rewards/rejected": -3.4090800285339355, |
| "step": 4690 |
| }, |
| { |
| "epoch": 1.71, |
| "grad_norm": 25.177344401658505, |
| "learning_rate": 9.659663914458913e-09, |
| "logits/chosen": -1.7834850549697876, |
| "logits/rejected": -1.7646923065185547, |
| "logps/chosen": -72.14763641357422, |
| "logps/rejected": -126.3357162475586, |
| "loss": 0.1295, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.4011167287826538, |
| "rewards/margins": 3.857720136642456, |
| "rewards/rejected": -2.4566032886505127, |
| "step": 4700 |
| }, |
| { |
| "epoch": 1.71, |
| "eval_logits/chosen": -2.4951701164245605, |
| "eval_logits/rejected": -2.3085877895355225, |
| "eval_logps/chosen": -93.2258071899414, |
| "eval_logps/rejected": -155.58546447753906, |
| "eval_loss": 0.25638025999069214, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.6730464100837708, |
| "eval_rewards/margins": 3.6944470405578613, |
| "eval_rewards/rejected": -3.0214006900787354, |
| "eval_runtime": 71.5377, |
| "eval_samples_per_second": 12.441, |
| "eval_steps_per_second": 0.196, |
| "step": 4700 |
| }, |
| { |
| "epoch": 1.71, |
| "grad_norm": 36.30094639660896, |
| "learning_rate": 9.427723227453092e-09, |
| "logits/chosen": -2.1383352279663086, |
| "logits/rejected": -1.7759078741073608, |
| "logps/chosen": -97.42940521240234, |
| "logps/rejected": -208.8566131591797, |
| "loss": 0.1838, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.45860522985458374, |
| "rewards/margins": 4.734647274017334, |
| "rewards/rejected": -4.276042461395264, |
| "step": 4710 |
| }, |
| { |
| "epoch": 1.71, |
| "grad_norm": 46.07295378551623, |
| "learning_rate": 9.198414203841732e-09, |
| "logits/chosen": -2.0975241661071777, |
| "logits/rejected": -1.4306576251983643, |
| "logps/chosen": -75.3366928100586, |
| "logps/rejected": -164.50413513183594, |
| "loss": 0.148, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5581697225570679, |
| "rewards/margins": 5.427392482757568, |
| "rewards/rejected": -3.869223117828369, |
| "step": 4720 |
| }, |
| { |
| "epoch": 1.72, |
| "grad_norm": 42.391432305833874, |
| "learning_rate": 8.971746046662982e-09, |
| "logits/chosen": -2.13923978805542, |
| "logits/rejected": -1.8475595712661743, |
| "logps/chosen": -85.93409729003906, |
| "logps/rejected": -238.55447387695312, |
| "loss": 0.1795, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.9917472004890442, |
| "rewards/margins": 4.265665054321289, |
| "rewards/rejected": -3.2739181518554688, |
| "step": 4730 |
| }, |
| { |
| "epoch": 1.72, |
| "grad_norm": 41.9386180805472, |
| "learning_rate": 8.747727852967013e-09, |
| "logits/chosen": -1.9976091384887695, |
| "logits/rejected": -1.6085163354873657, |
| "logps/chosen": -85.17452239990234, |
| "logps/rejected": -167.8629608154297, |
| "loss": 0.1809, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7708709239959717, |
| "rewards/margins": 4.317328929901123, |
| "rewards/rejected": -3.5464580059051514, |
| "step": 4740 |
| }, |
| { |
| "epoch": 1.72, |
| "grad_norm": 27.55797286092099, |
| "learning_rate": 8.526368613450938e-09, |
| "logits/chosen": -1.8709558248519897, |
| "logits/rejected": -1.3762298822402954, |
| "logps/chosen": -77.7394790649414, |
| "logps/rejected": -249.26089477539062, |
| "loss": 0.137, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0649783611297607, |
| "rewards/margins": 5.4183220863342285, |
| "rewards/rejected": -4.353343963623047, |
| "step": 4750 |
| }, |
| { |
| "epoch": 1.73, |
| "grad_norm": 37.03238003499655, |
| "learning_rate": 8.307677212098013e-09, |
| "logits/chosen": -2.055666446685791, |
| "logits/rejected": -1.7216275930404663, |
| "logps/chosen": -78.11017608642578, |
| "logps/rejected": -145.96435546875, |
| "loss": 0.1323, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.7488200664520264, |
| "rewards/margins": 4.145888328552246, |
| "rewards/rejected": -2.3970682621002197, |
| "step": 4760 |
| }, |
| { |
| "epoch": 1.73, |
| "grad_norm": 14.394014565941422, |
| "learning_rate": 8.091662425821027e-09, |
| "logits/chosen": -2.1242330074310303, |
| "logits/rejected": -1.989148736000061, |
| "logps/chosen": -96.9617691040039, |
| "logps/rejected": -156.48365783691406, |
| "loss": 0.146, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1661460399627686, |
| "rewards/margins": 4.207326889038086, |
| "rewards/rejected": -3.0411806106567383, |
| "step": 4770 |
| }, |
| { |
| "epoch": 1.74, |
| "grad_norm": 45.068535611865215, |
| "learning_rate": 7.878332924110114e-09, |
| "logits/chosen": -1.6985795497894287, |
| "logits/rejected": -1.4704097509384155, |
| "logps/chosen": -105.20362854003906, |
| "logps/rejected": -165.98757934570312, |
| "loss": 0.1946, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.41264334321022034, |
| "rewards/margins": 3.4486331939697266, |
| "rewards/rejected": -3.035989761352539, |
| "step": 4780 |
| }, |
| { |
| "epoch": 1.74, |
| "grad_norm": 32.28722287206916, |
| "learning_rate": 7.66769726868476e-09, |
| "logits/chosen": -2.389949321746826, |
| "logits/rejected": -1.8705673217773438, |
| "logps/chosen": -78.36787414550781, |
| "logps/rejected": -172.9830780029297, |
| "loss": 0.1543, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.4516584873199463, |
| "rewards/margins": 4.765595436096191, |
| "rewards/rejected": -3.313936948776245, |
| "step": 4790 |
| }, |
| { |
| "epoch": 1.74, |
| "grad_norm": 27.305664645978823, |
| "learning_rate": 7.459763913150232e-09, |
| "logits/chosen": -1.8216984272003174, |
| "logits/rejected": -1.5844495296478271, |
| "logps/chosen": -94.18133544921875, |
| "logps/rejected": -148.17559814453125, |
| "loss": 0.1277, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8979905843734741, |
| "rewards/margins": 4.38170051574707, |
| "rewards/rejected": -3.483710527420044, |
| "step": 4800 |
| }, |
| { |
| "epoch": 1.74, |
| "eval_logits/chosen": -2.5054409503936768, |
| "eval_logits/rejected": -2.3161699771881104, |
| "eval_logps/chosen": -93.1070327758789, |
| "eval_logps/rejected": -155.18020629882812, |
| "eval_loss": 0.2575148642063141, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.6849234700202942, |
| "eval_rewards/margins": 3.665799856185913, |
| "eval_rewards/rejected": -2.9808764457702637, |
| "eval_runtime": 71.4935, |
| "eval_samples_per_second": 12.449, |
| "eval_steps_per_second": 0.196, |
| "step": 4800 |
| }, |
| { |
| "epoch": 1.75, |
| "grad_norm": 33.21624232483828, |
| "learning_rate": 7.254541202658298e-09, |
| "logits/chosen": -1.866506576538086, |
| "logits/rejected": -1.7012319564819336, |
| "logps/chosen": -72.63794708251953, |
| "logps/rejected": -140.51022338867188, |
| "loss": 0.1671, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.9564884305000305, |
| "rewards/margins": 3.6187520027160645, |
| "rewards/rejected": -2.6622631549835205, |
| "step": 4810 |
| }, |
| { |
| "epoch": 1.75, |
| "grad_norm": 33.30055773008314, |
| "learning_rate": 7.052037373572247e-09, |
| "logits/chosen": -2.050233840942383, |
| "logits/rejected": -1.8784773349761963, |
| "logps/chosen": -76.64759826660156, |
| "logps/rejected": -121.65924072265625, |
| "loss": 0.1252, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1043304204940796, |
| "rewards/margins": 3.284348249435425, |
| "rewards/rejected": -2.1800179481506348, |
| "step": 4820 |
| }, |
| { |
| "epoch": 1.75, |
| "grad_norm": 30.226579991502298, |
| "learning_rate": 6.8522605531363995e-09, |
| "logits/chosen": -2.1329455375671387, |
| "logits/rejected": -1.5537135601043701, |
| "logps/chosen": -81.99913787841797, |
| "logps/rejected": -166.19219970703125, |
| "loss": 0.1238, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3520516157150269, |
| "rewards/margins": 4.49784517288208, |
| "rewards/rejected": -3.1457934379577637, |
| "step": 4830 |
| }, |
| { |
| "epoch": 1.76, |
| "grad_norm": 20.76586446781195, |
| "learning_rate": 6.655218759149936e-09, |
| "logits/chosen": -1.8627560138702393, |
| "logits/rejected": -1.4420944452285767, |
| "logps/chosen": -81.84294128417969, |
| "logps/rejected": -161.7127227783203, |
| "loss": 0.1383, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9598122835159302, |
| "rewards/margins": 3.9863998889923096, |
| "rewards/rejected": -3.026587963104248, |
| "step": 4840 |
| }, |
| { |
| "epoch": 1.76, |
| "grad_norm": 39.39440151277521, |
| "learning_rate": 6.460919899645045e-09, |
| "logits/chosen": -1.4853651523590088, |
| "logits/rejected": -1.0775649547576904, |
| "logps/chosen": -79.50499725341797, |
| "logps/rejected": -160.72036743164062, |
| "loss": 0.1273, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1241073608398438, |
| "rewards/margins": 4.072818756103516, |
| "rewards/rejected": -2.948711633682251, |
| "step": 4850 |
| }, |
| { |
| "epoch": 1.76, |
| "grad_norm": 28.117092220468667, |
| "learning_rate": 6.2693717725696064e-09, |
| "logits/chosen": -2.3298912048339844, |
| "logits/rejected": -1.7818387746810913, |
| "logps/chosen": -81.07538604736328, |
| "logps/rejected": -149.2225341796875, |
| "loss": 0.1653, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2655856609344482, |
| "rewards/margins": 4.464354038238525, |
| "rewards/rejected": -3.198768138885498, |
| "step": 4860 |
| }, |
| { |
| "epoch": 1.77, |
| "grad_norm": 27.103505885216574, |
| "learning_rate": 6.080582065474191e-09, |
| "logits/chosen": -1.792157769203186, |
| "logits/rejected": -1.6782718896865845, |
| "logps/chosen": -82.19720458984375, |
| "logps/rejected": -142.2840118408203, |
| "loss": 0.181, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.406231164932251, |
| "rewards/margins": 3.7717814445495605, |
| "rewards/rejected": -2.365550994873047, |
| "step": 4870 |
| }, |
| { |
| "epoch": 1.77, |
| "grad_norm": 30.313103414178183, |
| "learning_rate": 5.8945583552035664e-09, |
| "logits/chosen": -2.15095853805542, |
| "logits/rejected": -1.6400353908538818, |
| "logps/chosen": -76.02684783935547, |
| "logps/rejected": -157.7095947265625, |
| "loss": 0.1545, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7171390056610107, |
| "rewards/margins": 4.5480499267578125, |
| "rewards/rejected": -2.8309109210968018, |
| "step": 4880 |
| }, |
| { |
| "epoch": 1.77, |
| "grad_norm": 21.326128946676743, |
| "learning_rate": 5.711308107592575e-09, |
| "logits/chosen": -2.1534788608551025, |
| "logits/rejected": -1.7716478109359741, |
| "logps/chosen": -78.25006103515625, |
| "logps/rejected": -139.49354553222656, |
| "loss": 0.115, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0134289264678955, |
| "rewards/margins": 3.7863471508026123, |
| "rewards/rejected": -2.7729179859161377, |
| "step": 4890 |
| }, |
| { |
| "epoch": 1.78, |
| "grad_norm": 23.675706025969955, |
| "learning_rate": 5.530838677166514e-09, |
| "logits/chosen": -2.062622308731079, |
| "logits/rejected": -1.7914386987686157, |
| "logps/chosen": -71.51246643066406, |
| "logps/rejected": -151.47512817382812, |
| "loss": 0.1166, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6398112773895264, |
| "rewards/margins": 4.577850341796875, |
| "rewards/rejected": -2.9380390644073486, |
| "step": 4900 |
| }, |
| { |
| "epoch": 1.78, |
| "eval_logits/chosen": -2.498281478881836, |
| "eval_logits/rejected": -2.3098886013031006, |
| "eval_logps/chosen": -92.86518859863281, |
| "eval_logps/rejected": -155.0869140625, |
| "eval_loss": 0.25684645771980286, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7091068625450134, |
| "eval_rewards/margins": 3.680652618408203, |
| "eval_rewards/rejected": -2.971545934677124, |
| "eval_runtime": 71.3383, |
| "eval_samples_per_second": 12.476, |
| "eval_steps_per_second": 0.196, |
| "step": 4900 |
| }, |
| { |
| "epoch": 1.78, |
| "grad_norm": 38.69506032631391, |
| "learning_rate": 5.353157306845987e-09, |
| "logits/chosen": -1.927911400794983, |
| "logits/rejected": -1.6266618967056274, |
| "logps/chosen": -102.5804443359375, |
| "logps/rejected": -156.56808471679688, |
| "loss": 0.1485, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6019805669784546, |
| "rewards/margins": 3.804718017578125, |
| "rewards/rejected": -3.202737331390381, |
| "step": 4910 |
| }, |
| { |
| "epoch": 1.79, |
| "grad_norm": 13.185460602842793, |
| "learning_rate": 5.178271127656184e-09, |
| "logits/chosen": -1.6339209079742432, |
| "logits/rejected": -1.4754202365875244, |
| "logps/chosen": -77.33897399902344, |
| "logps/rejected": -137.8895721435547, |
| "loss": 0.1252, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3045456409454346, |
| "rewards/margins": 4.217949867248535, |
| "rewards/rejected": -2.9134037494659424, |
| "step": 4920 |
| }, |
| { |
| "epoch": 1.79, |
| "grad_norm": 19.230753940442828, |
| "learning_rate": 5.006187158440717e-09, |
| "logits/chosen": -1.7411714792251587, |
| "logits/rejected": -1.4912729263305664, |
| "logps/chosen": -79.78264617919922, |
| "logps/rejected": -136.29086303710938, |
| "loss": 0.1605, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1097663640975952, |
| "rewards/margins": 3.9273438453674316, |
| "rewards/rejected": -2.817577838897705, |
| "step": 4930 |
| }, |
| { |
| "epoch": 1.79, |
| "grad_norm": 46.07395788437603, |
| "learning_rate": 4.8369123055799295e-09, |
| "logits/chosen": -1.8338695764541626, |
| "logits/rejected": -1.4250398874282837, |
| "logps/chosen": -88.02108001708984, |
| "logps/rejected": -194.01132202148438, |
| "loss": 0.1362, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0556516647338867, |
| "rewards/margins": 5.144927024841309, |
| "rewards/rejected": -4.0892744064331055, |
| "step": 4940 |
| }, |
| { |
| "epoch": 1.8, |
| "grad_norm": 13.001369713867522, |
| "learning_rate": 4.67045336271368e-09, |
| "logits/chosen": -1.945007085800171, |
| "logits/rejected": -1.674750566482544, |
| "logps/chosen": -85.85972595214844, |
| "logps/rejected": -164.24606323242188, |
| "loss": 0.1525, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.063456654548645, |
| "rewards/margins": 4.070924758911133, |
| "rewards/rejected": -3.0074684619903564, |
| "step": 4950 |
| }, |
| { |
| "epoch": 1.8, |
| "grad_norm": 36.531934974949316, |
| "learning_rate": 4.506817010468731e-09, |
| "logits/chosen": -1.7169277667999268, |
| "logits/rejected": -1.929560899734497, |
| "logps/chosen": -106.58872985839844, |
| "logps/rejected": -153.10482788085938, |
| "loss": 0.1546, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9283292889595032, |
| "rewards/margins": 3.2342593669891357, |
| "rewards/rejected": -2.3059301376342773, |
| "step": 4960 |
| }, |
| { |
| "epoch": 1.8, |
| "grad_norm": 21.121804389411686, |
| "learning_rate": 4.346009816190596e-09, |
| "logits/chosen": -1.7104358673095703, |
| "logits/rejected": -1.638135313987732, |
| "logps/chosen": -96.46279907226562, |
| "logps/rejected": -162.83309936523438, |
| "loss": 0.1416, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8155636787414551, |
| "rewards/margins": 4.107938766479492, |
| "rewards/rejected": -3.292375087738037, |
| "step": 4970 |
| }, |
| { |
| "epoch": 1.81, |
| "grad_norm": 25.432800607703953, |
| "learning_rate": 4.188038233680005e-09, |
| "logits/chosen": -2.3670763969421387, |
| "logits/rejected": -1.915074348449707, |
| "logps/chosen": -76.34284210205078, |
| "logps/rejected": -146.34146118164062, |
| "loss": 0.1577, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.011473536491394, |
| "rewards/margins": 3.7747600078582764, |
| "rewards/rejected": -2.763286590576172, |
| "step": 4980 |
| }, |
| { |
| "epoch": 1.81, |
| "grad_norm": 31.52925521872735, |
| "learning_rate": 4.032908602933835e-09, |
| "logits/chosen": -1.9845958948135376, |
| "logits/rejected": -1.6368509531021118, |
| "logps/chosen": -78.15166473388672, |
| "logps/rejected": -183.4367218017578, |
| "loss": 0.1362, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.296852946281433, |
| "rewards/margins": 5.180371284484863, |
| "rewards/rejected": -3.8835182189941406, |
| "step": 4990 |
| }, |
| { |
| "epoch": 1.81, |
| "grad_norm": 24.267438377007352, |
| "learning_rate": 3.880627149890725e-09, |
| "logits/chosen": -1.933571219444275, |
| "logits/rejected": -1.8695943355560303, |
| "logps/chosen": -86.22135925292969, |
| "logps/rejected": -128.43087768554688, |
| "loss": 0.1273, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.3141534328460693, |
| "rewards/margins": 3.2799339294433594, |
| "rewards/rejected": -1.9657806158065796, |
| "step": 5000 |
| }, |
| { |
| "epoch": 1.81, |
| "eval_logits/chosen": -2.4975826740264893, |
| "eval_logits/rejected": -2.3100476264953613, |
| "eval_logps/chosen": -92.64019775390625, |
| "eval_logps/rejected": -155.10391235351562, |
| "eval_loss": 0.25643154978752136, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7316060066223145, |
| "eval_rewards/margins": 3.70485258102417, |
| "eval_rewards/rejected": -2.9732463359832764, |
| "eval_runtime": 71.5276, |
| "eval_samples_per_second": 12.443, |
| "eval_steps_per_second": 0.196, |
| "step": 5000 |
| }, |
| { |
| "epoch": 1.82, |
| "grad_norm": 23.454697098867037, |
| "learning_rate": 3.731199986181161e-09, |
| "logits/chosen": -1.693359375, |
| "logits/rejected": -1.7281681299209595, |
| "logps/chosen": -86.59770202636719, |
| "logps/rejected": -154.07138061523438, |
| "loss": 0.1429, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8508288264274597, |
| "rewards/margins": 3.944756031036377, |
| "rewards/rejected": -3.0939269065856934, |
| "step": 5010 |
| }, |
| { |
| "epoch": 1.82, |
| "grad_norm": 21.363949418831528, |
| "learning_rate": 3.5846331088821848e-09, |
| "logits/chosen": -1.7737480401992798, |
| "logits/rejected": -1.9281337261199951, |
| "logps/chosen": -82.63226318359375, |
| "logps/rejected": -151.45726013183594, |
| "loss": 0.1787, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.7113445401191711, |
| "rewards/margins": 3.4898841381073, |
| "rewards/rejected": -2.7785401344299316, |
| "step": 5020 |
| }, |
| { |
| "epoch": 1.83, |
| "grad_norm": 49.695952302649566, |
| "learning_rate": 3.440932400276758e-09, |
| "logits/chosen": -2.308803081512451, |
| "logits/rejected": -2.2135608196258545, |
| "logps/chosen": -83.8191909790039, |
| "logps/rejected": -141.08892822265625, |
| "loss": 0.1696, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.9245914220809937, |
| "rewards/margins": 3.2115063667297363, |
| "rewards/rejected": -2.286914825439453, |
| "step": 5030 |
| }, |
| { |
| "epoch": 1.83, |
| "grad_norm": 40.81068471888173, |
| "learning_rate": 3.300103627617656e-09, |
| "logits/chosen": -1.7834829092025757, |
| "logits/rejected": -1.9567530155181885, |
| "logps/chosen": -70.00810241699219, |
| "logps/rejected": -122.92684173583984, |
| "loss": 0.1558, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0573456287384033, |
| "rewards/margins": 4.120889186859131, |
| "rewards/rejected": -3.0635437965393066, |
| "step": 5040 |
| }, |
| { |
| "epoch": 1.83, |
| "grad_norm": 21.938277111569853, |
| "learning_rate": 3.162152442895996e-09, |
| "logits/chosen": -1.793460488319397, |
| "logits/rejected": -1.8442881107330322, |
| "logps/chosen": -101.73194885253906, |
| "logps/rejected": -148.40234375, |
| "loss": 0.1295, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0204694271087646, |
| "rewards/margins": 3.785799503326416, |
| "rewards/rejected": -2.7653300762176514, |
| "step": 5050 |
| }, |
| { |
| "epoch": 1.84, |
| "grad_norm": 37.435753201875194, |
| "learning_rate": 3.0270843826143837e-09, |
| "logits/chosen": -1.8540923595428467, |
| "logits/rejected": -1.9534356594085693, |
| "logps/chosen": -107.13368225097656, |
| "logps/rejected": -207.95645141601562, |
| "loss": 0.1731, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3544046878814697, |
| "rewards/margins": 4.78790807723999, |
| "rewards/rejected": -3.4335033893585205, |
| "step": 5060 |
| }, |
| { |
| "epoch": 1.84, |
| "grad_norm": 19.208193674331728, |
| "learning_rate": 2.894904867564793e-09, |
| "logits/chosen": -1.6729189157485962, |
| "logits/rejected": -1.791587471961975, |
| "logps/chosen": -92.18028259277344, |
| "logps/rejected": -181.7095947265625, |
| "loss": 0.1094, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.4680941104888916, |
| "rewards/margins": 5.646910190582275, |
| "rewards/rejected": -4.178815841674805, |
| "step": 5070 |
| }, |
| { |
| "epoch": 1.84, |
| "grad_norm": 29.97158214233071, |
| "learning_rate": 2.765619202610939e-09, |
| "logits/chosen": -1.8277852535247803, |
| "logits/rejected": -1.64214289188385, |
| "logps/chosen": -70.78485870361328, |
| "logps/rejected": -141.9372100830078, |
| "loss": 0.1758, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2488759756088257, |
| "rewards/margins": 3.880674362182617, |
| "rewards/rejected": -2.631798505783081, |
| "step": 5080 |
| }, |
| { |
| "epoch": 1.85, |
| "grad_norm": 17.74181700068005, |
| "learning_rate": 2.639232576475364e-09, |
| "logits/chosen": -1.552199125289917, |
| "logits/rejected": -1.4841490983963013, |
| "logps/chosen": -81.1534652709961, |
| "logps/rejected": -125.95042419433594, |
| "loss": 0.1427, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3184212446212769, |
| "rewards/margins": 3.3990256786346436, |
| "rewards/rejected": -2.080604314804077, |
| "step": 5090 |
| }, |
| { |
| "epoch": 1.85, |
| "grad_norm": 22.318813582652908, |
| "learning_rate": 2.5157500615312577e-09, |
| "logits/chosen": -1.8579572439193726, |
| "logits/rejected": -2.0184388160705566, |
| "logps/chosen": -89.17039489746094, |
| "logps/rejected": -150.26077270507812, |
| "loss": 0.1221, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.099916934967041, |
| "rewards/margins": 4.67440938949585, |
| "rewards/rejected": -3.5744922161102295, |
| "step": 5100 |
| }, |
| { |
| "epoch": 1.85, |
| "eval_logits/chosen": -2.479732036590576, |
| "eval_logits/rejected": -2.2966747283935547, |
| "eval_logps/chosen": -92.56282806396484, |
| "eval_logps/rejected": -155.0851593017578, |
| "eval_loss": 0.2575416564941406, |
| "eval_rewards/accuracies": 0.9107142686843872, |
| "eval_rewards/chosen": 0.7393438220024109, |
| "eval_rewards/margins": 3.7107155323028564, |
| "eval_rewards/rejected": -2.9713714122772217, |
| "eval_runtime": 71.474, |
| "eval_samples_per_second": 12.452, |
| "eval_steps_per_second": 0.196, |
| "step": 5100 |
| }, |
| { |
| "epoch": 1.85, |
| "grad_norm": 44.029689486297656, |
| "learning_rate": 2.395176613598815e-09, |
| "logits/chosen": -1.909277319908142, |
| "logits/rejected": -1.5398352146148682, |
| "logps/chosen": -100.70805358886719, |
| "logps/rejected": -156.3821563720703, |
| "loss": 0.1731, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.70123291015625, |
| "rewards/margins": 3.1760170459747314, |
| "rewards/rejected": -2.4747843742370605, |
| "step": 5110 |
| }, |
| { |
| "epoch": 1.86, |
| "grad_norm": 29.264329504378058, |
| "learning_rate": 2.2775170717463902e-09, |
| "logits/chosen": -2.2539279460906982, |
| "logits/rejected": -2.0020687580108643, |
| "logps/chosen": -96.94004821777344, |
| "logps/rejected": -158.09829711914062, |
| "loss": 0.1387, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7452301383018494, |
| "rewards/margins": 4.1389360427856445, |
| "rewards/rejected": -3.3937058448791504, |
| "step": 5120 |
| }, |
| { |
| "epoch": 1.86, |
| "grad_norm": 25.34055019816174, |
| "learning_rate": 2.1627761580962687e-09, |
| "logits/chosen": -2.1744818687438965, |
| "logits/rejected": -1.969853162765503, |
| "logps/chosen": -87.83367156982422, |
| "logps/rejected": -131.83323669433594, |
| "loss": 0.1646, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.7479078769683838, |
| "rewards/margins": 2.210872173309326, |
| "rewards/rejected": -1.4629642963409424, |
| "step": 5130 |
| }, |
| { |
| "epoch": 1.87, |
| "grad_norm": 28.764305762206437, |
| "learning_rate": 2.0509584776351506e-09, |
| "logits/chosen": -1.656760573387146, |
| "logits/rejected": -1.4122374057769775, |
| "logps/chosen": -72.59513092041016, |
| "logps/rejected": -148.3843994140625, |
| "loss": 0.1104, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1340370178222656, |
| "rewards/margins": 4.139061450958252, |
| "rewards/rejected": -3.0050246715545654, |
| "step": 5140 |
| }, |
| { |
| "epoch": 1.87, |
| "grad_norm": 25.193191046247687, |
| "learning_rate": 1.942068518029333e-09, |
| "logits/chosen": -2.1100473403930664, |
| "logits/rejected": -1.478046178817749, |
| "logps/chosen": -74.53004455566406, |
| "logps/rejected": -263.0050964355469, |
| "loss": 0.1563, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2791024446487427, |
| "rewards/margins": 4.334400177001953, |
| "rewards/rejected": -3.055298089981079, |
| "step": 5150 |
| }, |
| { |
| "epoch": 1.87, |
| "grad_norm": 38.53224961606162, |
| "learning_rate": 1.8361106494445943e-09, |
| "logits/chosen": -2.0669713020324707, |
| "logits/rejected": -1.703884482383728, |
| "logps/chosen": -94.36878967285156, |
| "logps/rejected": -157.6041717529297, |
| "loss": 0.1447, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8586713671684265, |
| "rewards/margins": 3.7407805919647217, |
| "rewards/rejected": -2.8821091651916504, |
| "step": 5160 |
| }, |
| { |
| "epoch": 1.88, |
| "grad_norm": 43.063554524576524, |
| "learning_rate": 1.7330891243708456e-09, |
| "logits/chosen": -1.6464707851409912, |
| "logits/rejected": -1.8101832866668701, |
| "logps/chosen": -121.72802734375, |
| "logps/rejected": -158.669677734375, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.5665308237075806, |
| "rewards/margins": 3.349714756011963, |
| "rewards/rejected": -2.7831835746765137, |
| "step": 5170 |
| }, |
| { |
| "epoch": 1.88, |
| "grad_norm": 40.989160241743704, |
| "learning_rate": 1.633008077451406e-09, |
| "logits/chosen": -2.263235569000244, |
| "logits/rejected": -2.035400867462158, |
| "logps/chosen": -78.44254302978516, |
| "logps/rejected": -152.73580932617188, |
| "loss": 0.1752, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6941092014312744, |
| "rewards/margins": 4.51024055480957, |
| "rewards/rejected": -2.816131353378296, |
| "step": 5180 |
| }, |
| { |
| "epoch": 1.88, |
| "grad_norm": 35.64059838178094, |
| "learning_rate": 1.5358715253170785e-09, |
| "logits/chosen": -1.6528642177581787, |
| "logits/rejected": -1.7316625118255615, |
| "logps/chosen": -97.46934509277344, |
| "logps/rejected": -128.89291381835938, |
| "loss": 0.1489, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.1307111978530884, |
| "rewards/margins": 3.048318386077881, |
| "rewards/rejected": -1.917607069015503, |
| "step": 5190 |
| }, |
| { |
| "epoch": 1.89, |
| "grad_norm": 21.32011780611291, |
| "learning_rate": 1.4416833664249867e-09, |
| "logits/chosen": -1.8180415630340576, |
| "logits/rejected": -1.7365614175796509, |
| "logps/chosen": -99.86917877197266, |
| "logps/rejected": -146.1660919189453, |
| "loss": 0.1752, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.311732530593872, |
| "rewards/margins": 3.4762260913848877, |
| "rewards/rejected": -2.1644930839538574, |
| "step": 5200 |
| }, |
| { |
| "epoch": 1.89, |
| "eval_logits/chosen": -2.4827630519866943, |
| "eval_logits/rejected": -2.2978460788726807, |
| "eval_logps/chosen": -92.51727294921875, |
| "eval_logps/rejected": -155.1031036376953, |
| "eval_loss": 0.2567766308784485, |
| "eval_rewards/accuracies": 0.9107142686843872, |
| "eval_rewards/chosen": 0.7438983917236328, |
| "eval_rewards/margins": 3.717064380645752, |
| "eval_rewards/rejected": -2.973165512084961, |
| "eval_runtime": 71.4535, |
| "eval_samples_per_second": 12.456, |
| "eval_steps_per_second": 0.196, |
| "step": 5200 |
| }, |
| { |
| "epoch": 1.89, |
| "grad_norm": 14.906342368048502, |
| "learning_rate": 1.3504473809020673e-09, |
| "logits/chosen": -1.907004714012146, |
| "logits/rejected": -1.4068089723587036, |
| "logps/chosen": -86.4648208618164, |
| "logps/rejected": -206.89736938476562, |
| "loss": 0.1684, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7831238508224487, |
| "rewards/margins": 4.615767478942871, |
| "rewards/rejected": -3.8326430320739746, |
| "step": 5210 |
| }, |
| { |
| "epoch": 1.89, |
| "grad_norm": 34.29482897768453, |
| "learning_rate": 1.2621672303933739e-09, |
| "logits/chosen": -1.811163306236267, |
| "logits/rejected": -1.6804473400115967, |
| "logps/chosen": -95.26229858398438, |
| "logps/rejected": -157.65988159179688, |
| "loss": 0.1628, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.0265977382659912, |
| "rewards/margins": 3.7031357288360596, |
| "rewards/rejected": -2.6765377521514893, |
| "step": 5220 |
| }, |
| { |
| "epoch": 1.9, |
| "grad_norm": 28.11537513046596, |
| "learning_rate": 1.1768464579151373e-09, |
| "logits/chosen": -1.8990215063095093, |
| "logits/rejected": -1.9127355813980103, |
| "logps/chosen": -82.57646179199219, |
| "logps/rejected": -136.97134399414062, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.8750249743461609, |
| "rewards/margins": 3.5186927318573, |
| "rewards/rejected": -2.643667697906494, |
| "step": 5230 |
| }, |
| { |
| "epoch": 1.9, |
| "grad_norm": 9.966571219874533, |
| "learning_rate": 1.0944884877125527e-09, |
| "logits/chosen": -1.402052640914917, |
| "logits/rejected": -1.1487385034561157, |
| "logps/chosen": -104.62020111083984, |
| "logps/rejected": -200.74160766601562, |
| "loss": 0.1199, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.388352632522583, |
| "rewards/margins": 4.531402111053467, |
| "rewards/rejected": -4.143049716949463, |
| "step": 5240 |
| }, |
| { |
| "epoch": 1.91, |
| "grad_norm": 20.5792937534979, |
| "learning_rate": 1.0150966251223664e-09, |
| "logits/chosen": -2.0096936225891113, |
| "logits/rejected": -1.805040717124939, |
| "logps/chosen": -64.17274475097656, |
| "logps/rejected": -122.17179870605469, |
| "loss": 0.1306, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.0267283916473389, |
| "rewards/margins": 4.174807071685791, |
| "rewards/rejected": -3.1480789184570312, |
| "step": 5250 |
| }, |
| { |
| "epoch": 1.91, |
| "grad_norm": 20.585526219504107, |
| "learning_rate": 9.386740564401808e-10, |
| "logits/chosen": -2.2588255405426025, |
| "logits/rejected": -1.8842235803604126, |
| "logps/chosen": -115.33465576171875, |
| "logps/rejected": -176.01412963867188, |
| "loss": 0.1458, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.2555707693099976, |
| "rewards/margins": 4.751716613769531, |
| "rewards/rejected": -3.4961459636688232, |
| "step": 5260 |
| }, |
| { |
| "epoch": 1.91, |
| "grad_norm": 34.226305800359405, |
| "learning_rate": 8.652238487926566e-10, |
| "logits/chosen": -1.9869095087051392, |
| "logits/rejected": -2.1987595558166504, |
| "logps/chosen": -92.55632781982422, |
| "logps/rejected": -212.2284393310547, |
| "loss": 0.1584, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0699260234832764, |
| "rewards/margins": 3.5520572662353516, |
| "rewards/rejected": -2.4821314811706543, |
| "step": 5270 |
| }, |
| { |
| "epoch": 1.92, |
| "grad_norm": 36.554661458368045, |
| "learning_rate": 7.947489500143206e-10, |
| "logits/chosen": -2.079451560974121, |
| "logits/rejected": -1.7273361682891846, |
| "logps/chosen": -59.18944549560547, |
| "logps/rejected": -125.39189147949219, |
| "loss": 0.1414, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8779332637786865, |
| "rewards/margins": 3.9232311248779297, |
| "rewards/rejected": -2.045297622680664, |
| "step": 5280 |
| }, |
| { |
| "epoch": 1.92, |
| "grad_norm": 29.3453328036471, |
| "learning_rate": 7.272521885293343e-10, |
| "logits/chosen": -1.7185804843902588, |
| "logits/rejected": -1.2968146800994873, |
| "logps/chosen": -69.44091033935547, |
| "logps/rejected": -128.18624877929688, |
| "loss": 0.1573, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.9322481155395508, |
| "rewards/margins": 3.3514316082000732, |
| "rewards/rejected": -2.4191837310791016, |
| "step": 5290 |
| }, |
| { |
| "epoch": 1.92, |
| "grad_norm": 19.154161212924226, |
| "learning_rate": 6.627362732379433e-10, |
| "logits/chosen": -1.8239357471466064, |
| "logits/rejected": -1.7603832483291626, |
| "logps/chosen": -96.85991668701172, |
| "logps/rejected": -174.81370544433594, |
| "loss": 0.1235, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8014510869979858, |
| "rewards/margins": 3.704523801803589, |
| "rewards/rejected": -2.9030728340148926, |
| "step": 5300 |
| }, |
| { |
| "epoch": 1.92, |
| "eval_logits/chosen": -2.494673490524292, |
| "eval_logits/rejected": -2.307727336883545, |
| "eval_logps/chosen": -92.54339599609375, |
| "eval_logps/rejected": -155.1482696533203, |
| "eval_loss": 0.25662314891815186, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7412868142127991, |
| "eval_rewards/margins": 3.718968629837036, |
| "eval_rewards/rejected": -2.97768235206604, |
| "eval_runtime": 71.3827, |
| "eval_samples_per_second": 12.468, |
| "eval_steps_per_second": 0.196, |
| "step": 5300 |
| }, |
| { |
| "epoch": 1.93, |
| "grad_norm": 37.14417488124076, |
| "learning_rate": 6.012037934077563e-10, |
| "logits/chosen": -1.4998286962509155, |
| "logits/rejected": -1.2366199493408203, |
| "logps/chosen": -93.3680648803711, |
| "logps/rejected": -167.04367065429688, |
| "loss": 0.148, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9477823972702026, |
| "rewards/margins": 3.8279433250427246, |
| "rewards/rejected": -2.8801610469818115, |
| "step": 5310 |
| }, |
| { |
| "epoch": 1.93, |
| "grad_norm": 36.70052172928171, |
| "learning_rate": 5.426572185698363e-10, |
| "logits/chosen": -2.037949323654175, |
| "logits/rejected": -1.7177932262420654, |
| "logps/chosen": -88.7237548828125, |
| "logps/rejected": -163.99179077148438, |
| "loss": 0.1393, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0409705638885498, |
| "rewards/margins": 3.7800567150115967, |
| "rewards/rejected": -2.7390859127044678, |
| "step": 5320 |
| }, |
| { |
| "epoch": 1.93, |
| "grad_norm": 57.51895033093402, |
| "learning_rate": 4.870988984196134e-10, |
| "logits/chosen": -1.5953149795532227, |
| "logits/rejected": -1.3189541101455688, |
| "logps/chosen": -92.99564361572266, |
| "logps/rejected": -165.5485382080078, |
| "loss": 0.1624, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.016343355178833, |
| "rewards/margins": 4.139420509338379, |
| "rewards/rejected": -3.123077392578125, |
| "step": 5330 |
| }, |
| { |
| "epoch": 1.94, |
| "grad_norm": 29.612269350826534, |
| "learning_rate": 4.345310627225179e-10, |
| "logits/chosen": -2.3673906326293945, |
| "logits/rejected": -1.5509955883026123, |
| "logps/chosen": -72.61958312988281, |
| "logps/rejected": -196.7791290283203, |
| "loss": 0.1208, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.651362419128418, |
| "rewards/margins": 6.749650478363037, |
| "rewards/rejected": -5.098288536071777, |
| "step": 5340 |
| }, |
| { |
| "epoch": 1.94, |
| "grad_norm": 13.368844959539826, |
| "learning_rate": 3.849558212245696e-10, |
| "logits/chosen": -1.5998425483703613, |
| "logits/rejected": -1.4031832218170166, |
| "logps/chosen": -85.34275817871094, |
| "logps/rejected": -154.41502380371094, |
| "loss": 0.1364, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.0656564235687256, |
| "rewards/margins": 3.6739463806152344, |
| "rewards/rejected": -2.608290195465088, |
| "step": 5350 |
| }, |
| { |
| "epoch": 1.95, |
| "grad_norm": 32.464438755302695, |
| "learning_rate": 3.3837516356764464e-10, |
| "logits/chosen": -2.0724587440490723, |
| "logits/rejected": -1.6080551147460938, |
| "logps/chosen": -91.76570129394531, |
| "logps/rejected": -153.9037628173828, |
| "loss": 0.1588, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.46443551778793335, |
| "rewards/margins": 2.967500925064087, |
| "rewards/rejected": -2.503065586090088, |
| "step": 5360 |
| }, |
| { |
| "epoch": 1.95, |
| "grad_norm": 17.335570454742644, |
| "learning_rate": 2.947909592096648e-10, |
| "logits/chosen": -1.7819137573242188, |
| "logits/rejected": -1.6901859045028687, |
| "logps/chosen": -83.9917221069336, |
| "logps/rejected": -155.39439392089844, |
| "loss": 0.17, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1832668781280518, |
| "rewards/margins": 4.525895118713379, |
| "rewards/rejected": -3.3426289558410645, |
| "step": 5370 |
| }, |
| { |
| "epoch": 1.95, |
| "grad_norm": 24.93213979968397, |
| "learning_rate": 2.542049573495325e-10, |
| "logits/chosen": -1.923814058303833, |
| "logits/rejected": -1.7243621349334717, |
| "logps/chosen": -109.0640869140625, |
| "logps/rejected": -164.2493438720703, |
| "loss": 0.1342, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.23761026561260223, |
| "rewards/margins": 3.5649254322052, |
| "rewards/rejected": -3.327315092086792, |
| "step": 5380 |
| }, |
| { |
| "epoch": 1.96, |
| "grad_norm": 25.680467090159638, |
| "learning_rate": 2.166187868569619e-10, |
| "logits/chosen": -1.646674394607544, |
| "logits/rejected": -1.8654903173446655, |
| "logps/chosen": -107.28630065917969, |
| "logps/rejected": -148.73915100097656, |
| "loss": 0.1411, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.48937278985977173, |
| "rewards/margins": 3.2133946418762207, |
| "rewards/rejected": -2.7240214347839355, |
| "step": 5390 |
| }, |
| { |
| "epoch": 1.96, |
| "grad_norm": 26.62093298807946, |
| "learning_rate": 1.8203395620708107e-10, |
| "logits/chosen": -2.068347215652466, |
| "logits/rejected": -1.635754942893982, |
| "logps/chosen": -80.6448745727539, |
| "logps/rejected": -183.50900268554688, |
| "loss": 0.1288, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.8882523775100708, |
| "rewards/margins": 4.448118209838867, |
| "rewards/rejected": -3.559866428375244, |
| "step": 5400 |
| }, |
| { |
| "epoch": 1.96, |
| "eval_logits/chosen": -2.488870143890381, |
| "eval_logits/rejected": -2.304009199142456, |
| "eval_logps/chosen": -92.57149505615234, |
| "eval_logps/rejected": -155.0608673095703, |
| "eval_loss": 0.2569134533405304, |
| "eval_rewards/accuracies": 0.9107142686843872, |
| "eval_rewards/chosen": 0.738475501537323, |
| "eval_rewards/margins": 3.7074153423309326, |
| "eval_rewards/rejected": -2.968940258026123, |
| "eval_runtime": 71.568, |
| "eval_samples_per_second": 12.436, |
| "eval_steps_per_second": 0.196, |
| "step": 5400 |
| }, |
| { |
| "epoch": 1.96, |
| "grad_norm": 60.89983951450628, |
| "learning_rate": 1.5045185341992228e-10, |
| "logits/chosen": -2.275635242462158, |
| "logits/rejected": -2.2029271125793457, |
| "logps/chosen": -95.09009552001953, |
| "logps/rejected": -132.654296875, |
| "loss": 0.1563, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.0345149040222168, |
| "rewards/margins": 3.2276673316955566, |
| "rewards/rejected": -2.193152666091919, |
| "step": 5410 |
| }, |
| { |
| "epoch": 1.97, |
| "grad_norm": 42.30577726329947, |
| "learning_rate": 1.218737460046748e-10, |
| "logits/chosen": -1.8863006830215454, |
| "logits/rejected": -1.45746910572052, |
| "logps/chosen": -86.98088836669922, |
| "logps/rejected": -162.14358520507812, |
| "loss": 0.1412, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 1.4029314517974854, |
| "rewards/margins": 4.1309075355529785, |
| "rewards/rejected": -2.727975368499756, |
| "step": 5420 |
| }, |
| { |
| "epoch": 1.97, |
| "grad_norm": 22.851908976691202, |
| "learning_rate": 9.630078090883398e-11, |
| "logits/chosen": -2.069901943206787, |
| "logits/rejected": -1.7926028966903687, |
| "logps/chosen": -121.98948669433594, |
| "logps/rejected": -309.41168212890625, |
| "loss": 0.1431, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.005540943238884211, |
| "rewards/margins": 4.165543556213379, |
| "rewards/rejected": -4.171084403991699, |
| "step": 5430 |
| }, |
| { |
| "epoch": 1.97, |
| "grad_norm": 24.450065176253204, |
| "learning_rate": 7.373398447218792e-11, |
| "logits/chosen": -1.7372915744781494, |
| "logits/rejected": -1.5503891706466675, |
| "logps/chosen": -108.78375244140625, |
| "logps/rejected": -175.95468139648438, |
| "loss": 0.1728, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1438463926315308, |
| "rewards/margins": 4.212247848510742, |
| "rewards/rejected": -3.06840181350708, |
| "step": 5440 |
| }, |
| { |
| "epoch": 1.98, |
| "grad_norm": 13.790530961269235, |
| "learning_rate": 5.417426238560896e-11, |
| "logits/chosen": -1.820669412612915, |
| "logits/rejected": -1.7920053005218506, |
| "logps/chosen": -83.18289184570312, |
| "logps/rejected": -164.08082580566406, |
| "loss": 0.1263, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.9669841527938843, |
| "rewards/margins": 4.896847724914551, |
| "rewards/rejected": -3.9298641681671143, |
| "step": 5450 |
| }, |
| { |
| "epoch": 1.98, |
| "grad_norm": 21.370222843228014, |
| "learning_rate": 3.7622399654682614e-11, |
| "logits/chosen": -2.2546284198760986, |
| "logits/rejected": -1.9654285907745361, |
| "logps/chosen": -91.28435516357422, |
| "logps/rejected": -227.45193481445312, |
| "loss": 0.1813, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.5948358774185181, |
| "rewards/margins": 4.2998552322387695, |
| "rewards/rejected": -3.705019474029541, |
| "step": 5460 |
| }, |
| { |
| "epoch": 1.99, |
| "grad_norm": 19.65088303473795, |
| "learning_rate": 2.4079060568257813e-11, |
| "logits/chosen": -1.8808691501617432, |
| "logits/rejected": -1.6000111103057861, |
| "logps/chosen": -88.05937957763672, |
| "logps/rejected": -188.5884552001953, |
| "loss": 0.1328, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3033487796783447, |
| "rewards/margins": 4.862802982330322, |
| "rewards/rejected": -3.5594539642333984, |
| "step": 5470 |
| }, |
| { |
| "epoch": 1.99, |
| "grad_norm": 27.89489697549713, |
| "learning_rate": 1.354478867173492e-11, |
| "logits/chosen": -1.8578226566314697, |
| "logits/rejected": -1.4207289218902588, |
| "logps/chosen": -113.3355941772461, |
| "logps/rejected": -168.69332885742188, |
| "loss": 0.1488, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.764236330986023, |
| "rewards/margins": 3.6643853187561035, |
| "rewards/rejected": -2.900148868560791, |
| "step": 5480 |
| }, |
| { |
| "epoch": 1.99, |
| "grad_norm": 20.734274017785214, |
| "learning_rate": 6.020006745274808e-12, |
| "logits/chosen": -2.122195243835449, |
| "logits/rejected": -1.6640615463256836, |
| "logps/chosen": -59.822662353515625, |
| "logps/rejected": -172.18661499023438, |
| "loss": 0.1118, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.6453478336334229, |
| "rewards/margins": 4.6514892578125, |
| "rewards/rejected": -3.006141185760498, |
| "step": 5490 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 45.552319616907596, |
| "learning_rate": 1.5050167868208009e-12, |
| "logits/chosen": -1.6916072368621826, |
| "logits/rejected": -1.7415540218353271, |
| "logps/chosen": -121.29073333740234, |
| "logps/rejected": -157.93690490722656, |
| "loss": 0.145, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.7984371185302734, |
| "rewards/margins": 4.016904354095459, |
| "rewards/rejected": -3.2184669971466064, |
| "step": 5500 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_logits/chosen": -2.4879519939422607, |
| "eval_logits/rejected": -2.303224802017212, |
| "eval_logps/chosen": -92.59040069580078, |
| "eval_logps/rejected": -155.1884002685547, |
| "eval_loss": 0.2571623623371124, |
| "eval_rewards/accuracies": 0.8928571343421936, |
| "eval_rewards/chosen": 0.7365875840187073, |
| "eval_rewards/margins": 3.7182834148406982, |
| "eval_rewards/rejected": -2.9816958904266357, |
| "eval_runtime": 71.3635, |
| "eval_samples_per_second": 12.471, |
| "eval_steps_per_second": 0.196, |
| "step": 5500 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 18.200273756614273, |
| "learning_rate": 0.0, |
| "logits/chosen": -1.8841540813446045, |
| "logits/rejected": -1.5108808279037476, |
| "logps/chosen": -88.10948181152344, |
| "logps/rejected": -193.3212127685547, |
| "loss": 0.1231, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 1.3105828762054443, |
| "rewards/margins": 4.1531171798706055, |
| "rewards/rejected": -2.842533826828003, |
| "step": 5510 |
| }, |
| { |
| "epoch": 2.0, |
| "step": 5510, |
| "total_flos": 0.0, |
| "train_loss": 0.2711078498627443, |
| "train_runtime": 49070.973, |
| "train_samples_per_second": 3.592, |
| "train_steps_per_second": 0.112 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 5510, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 50, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
