| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 60, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.6346618384122849, |
| "epoch": 0.050314465408805034, |
| "grad_norm": 3.734375, |
| "learning_rate": 5e-05, |
| "logits/chosen": -0.7411658949470903, |
| "logits/rejected": -0.20507352810005436, |
| "logps/chosen": -152.59497356414795, |
| "logps/rejected": -195.90787506103516, |
| "loss": 0.6931471824645996, |
| "mean_token_accuracy": 0.562163695693016, |
| "num_tokens": 6930.0, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "entropy": 1.4584085196256638, |
| "epoch": 0.10062893081761007, |
| "grad_norm": 3.40625, |
| "learning_rate": 4.9166666666666665e-05, |
| "logits/chosen": -0.9251237438184291, |
| "logits/rejected": -0.42817166651451355, |
| "logps/chosen": -153.27350616455078, |
| "logps/rejected": -180.06759071350098, |
| "loss": 0.5006560683250427, |
| "mean_token_accuracy": 0.583746887743473, |
| "num_tokens": 12841.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.23006458766758442, |
| "rewards/margins": 0.4357452392578125, |
| "rewards/rejected": -0.20568065904080868, |
| "step": 2 |
| }, |
| { |
| "entropy": 1.4456398040056229, |
| "epoch": 0.1509433962264151, |
| "grad_norm": 2.609375, |
| "learning_rate": 4.8333333333333334e-05, |
| "logits/chosen": -1.0732471831941308, |
| "logits/rejected": -0.5970107323243525, |
| "logps/chosen": -146.46685123443604, |
| "logps/rejected": -187.66933250427246, |
| "loss": 0.3808254599571228, |
| "mean_token_accuracy": 0.6039908826351166, |
| "num_tokens": 18816.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.34212866611778736, |
| "rewards/margins": 0.7833537347614765, |
| "rewards/rejected": -0.4412250593304634, |
| "step": 3 |
| }, |
| { |
| "entropy": 1.5161051750183105, |
| "epoch": 0.20125786163522014, |
| "grad_norm": 2.15625, |
| "learning_rate": 4.75e-05, |
| "logits/chosen": -0.9412079692678976, |
| "logits/rejected": -0.4002159728435217, |
| "logps/chosen": -118.29372596740723, |
| "logps/rejected": -182.18238067626953, |
| "loss": 0.34213775396347046, |
| "mean_token_accuracy": 0.5912635922431946, |
| "num_tokens": 25522.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.3642648714594543, |
| "rewards/margins": 0.954810731112957, |
| "rewards/rejected": -0.5905458554625511, |
| "step": 4 |
| }, |
| { |
| "entropy": 1.4330662935972214, |
| "epoch": 0.25157232704402516, |
| "grad_norm": 1.7421875, |
| "learning_rate": 4.666666666666667e-05, |
| "logits/chosen": -1.2388932583587922, |
| "logits/rejected": -0.6178736694127079, |
| "logps/chosen": -118.57859134674072, |
| "logps/rejected": -193.721284866333, |
| "loss": 0.21044230461120605, |
| "mean_token_accuracy": 0.5765212289988995, |
| "num_tokens": 32576.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.45518894493579865, |
| "rewards/margins": 1.520921140909195, |
| "rewards/rejected": -1.065732218325138, |
| "step": 5 |
| }, |
| { |
| "entropy": 1.4573922604322433, |
| "epoch": 0.3018867924528302, |
| "grad_norm": 1.484375, |
| "learning_rate": 4.5833333333333334e-05, |
| "logits/chosen": -1.2737800530058234, |
| "logits/rejected": -0.5034645169102513, |
| "logps/chosen": -120.18647003173828, |
| "logps/rejected": -203.72421646118164, |
| "loss": 0.17699970304965973, |
| "mean_token_accuracy": 0.5985999256372452, |
| "num_tokens": 39703.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.41289406828582287, |
| "rewards/margins": 1.7254902124404907, |
| "rewards/rejected": -1.3125961422920227, |
| "step": 6 |
| }, |
| { |
| "entropy": 1.2998351603746414, |
| "epoch": 0.3522012578616352, |
| "grad_norm": 1.546875, |
| "learning_rate": 4.5e-05, |
| "logits/chosen": -1.3130657002820167, |
| "logits/rejected": -0.6094413558435448, |
| "logps/chosen": -108.3181095123291, |
| "logps/rejected": -194.44207382202148, |
| "loss": 0.18618769943714142, |
| "mean_token_accuracy": 0.6349928826093674, |
| "num_tokens": 47481.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.26029996760189533, |
| "rewards/margins": 1.7353856414556503, |
| "rewards/rejected": -1.4750856757164001, |
| "step": 7 |
| }, |
| { |
| "entropy": 1.354924201965332, |
| "epoch": 0.4025157232704403, |
| "grad_norm": 1.015625, |
| "learning_rate": 4.4166666666666665e-05, |
| "logits/chosen": -1.437566920334179, |
| "logits/rejected": -1.0028015186250743, |
| "logps/chosen": -137.25361728668213, |
| "logps/rejected": -203.34245681762695, |
| "loss": 0.1016073077917099, |
| "mean_token_accuracy": 0.6037986874580383, |
| "num_tokens": 54551.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6320773344486952, |
| "rewards/margins": 2.5685721784830093, |
| "rewards/rejected": -1.9364948570728302, |
| "step": 8 |
| }, |
| { |
| "entropy": 1.3007782101631165, |
| "epoch": 0.4528301886792453, |
| "grad_norm": 0.6796875, |
| "learning_rate": 4.3333333333333334e-05, |
| "logits/chosen": -1.4337730887270794, |
| "logits/rejected": -0.9420388615307833, |
| "logps/chosen": -133.88286685943604, |
| "logps/rejected": -215.5273036956787, |
| "loss": 0.06624128669500351, |
| "mean_token_accuracy": 0.6150126904249191, |
| "num_tokens": 61082.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7177156624384224, |
| "rewards/margins": 3.2935406416654587, |
| "rewards/rejected": -2.5758249759674072, |
| "step": 9 |
| }, |
| { |
| "entropy": 1.3091522455215454, |
| "epoch": 0.5031446540880503, |
| "grad_norm": 0.515625, |
| "learning_rate": 4.25e-05, |
| "logits/chosen": -1.7237460889408442, |
| "logits/rejected": -1.1708205992219858, |
| "logps/chosen": -114.79162216186523, |
| "logps/rejected": -216.33137321472168, |
| "loss": 0.03747009485960007, |
| "mean_token_accuracy": 0.5977272689342499, |
| "num_tokens": 68063.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7877620831131935, |
| "rewards/margins": 4.083773195743561, |
| "rewards/rejected": -3.296011045575142, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.2404007613658905, |
| "epoch": 0.5534591194968553, |
| "grad_norm": 0.259765625, |
| "learning_rate": 4.166666666666667e-05, |
| "logits/chosen": -1.655923360923379, |
| "logits/rejected": -1.1929389227396723, |
| "logps/chosen": -137.04386043548584, |
| "logps/rejected": -223.17731285095215, |
| "loss": 0.019634969532489777, |
| "mean_token_accuracy": 0.615639328956604, |
| "num_tokens": 73558.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1441535080084577, |
| "rewards/margins": 4.826931297779083, |
| "rewards/rejected": -3.6827778220176697, |
| "step": 11 |
| }, |
| { |
| "entropy": 1.2675090283155441, |
| "epoch": 0.6037735849056604, |
| "grad_norm": 0.16796875, |
| "learning_rate": 4.0833333333333334e-05, |
| "logits/chosen": -1.738670325722853, |
| "logits/rejected": -1.2716987398570991, |
| "logps/chosen": -133.57194137573242, |
| "logps/rejected": -223.28443908691406, |
| "loss": 0.013121644034981728, |
| "mean_token_accuracy": 0.6074652224779129, |
| "num_tokens": 79421.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0685334280133247, |
| "rewards/margins": 5.048604816198349, |
| "rewards/rejected": -3.980071395635605, |
| "step": 12 |
| }, |
| { |
| "entropy": 1.221432313323021, |
| "epoch": 0.6540880503144654, |
| "grad_norm": 0.466796875, |
| "learning_rate": 4e-05, |
| "logits/chosen": -1.8638311019017777, |
| "logits/rejected": -1.3540506586835501, |
| "logps/chosen": -130.8041524887085, |
| "logps/rejected": -240.23817443847656, |
| "loss": 0.029251903295516968, |
| "mean_token_accuracy": 0.6208610609173775, |
| "num_tokens": 86682.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.4837151520187035, |
| "rewards/margins": 5.231908708810806, |
| "rewards/rejected": -4.748193636536598, |
| "step": 13 |
| }, |
| { |
| "entropy": 1.179955169558525, |
| "epoch": 0.7044025157232704, |
| "grad_norm": 0.263671875, |
| "learning_rate": 3.9166666666666665e-05, |
| "logits/chosen": -1.714330251919895, |
| "logits/rejected": -1.4322710140173394, |
| "logps/chosen": -134.82413864135742, |
| "logps/rejected": -221.86817169189453, |
| "loss": 0.019698552787303925, |
| "mean_token_accuracy": 0.6337382346391678, |
| "num_tokens": 93035.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.23185482621192932, |
| "rewards/margins": 5.084479838609695, |
| "rewards/rejected": -4.852624952793121, |
| "step": 14 |
| }, |
| { |
| "entropy": 1.1727432310581207, |
| "epoch": 0.7547169811320755, |
| "grad_norm": 0.046142578125, |
| "learning_rate": 3.8333333333333334e-05, |
| "logits/chosen": -1.907136892939249, |
| "logits/rejected": -1.514815267146274, |
| "logps/chosen": -132.96124839782715, |
| "logps/rejected": -249.50230979919434, |
| "loss": 0.002145277801901102, |
| "mean_token_accuracy": 0.6479089632630348, |
| "num_tokens": 98572.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1812188103795052, |
| "rewards/margins": 7.1862099170684814, |
| "rewards/rejected": -6.004990994930267, |
| "step": 15 |
| }, |
| { |
| "entropy": 1.2092190980911255, |
| "epoch": 0.8050314465408805, |
| "grad_norm": 0.203125, |
| "learning_rate": 3.7500000000000003e-05, |
| "logits/chosen": -1.8553556408587935, |
| "logits/rejected": -1.4260795074385058, |
| "logps/chosen": -139.758376121521, |
| "logps/rejected": -248.12162590026855, |
| "loss": 0.015183546580374241, |
| "mean_token_accuracy": 0.6128971949219704, |
| "num_tokens": 106014.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.30653896857984364, |
| "rewards/margins": 6.243123233318329, |
| "rewards/rejected": -5.9365842044353485, |
| "step": 16 |
| }, |
| { |
| "entropy": 1.2008470296859741, |
| "epoch": 0.8553459119496856, |
| "grad_norm": 0.134765625, |
| "learning_rate": 3.6666666666666666e-05, |
| "logits/chosen": -1.9781900924423292, |
| "logits/rejected": -1.3513694447951352, |
| "logps/chosen": -127.7047929763794, |
| "logps/rejected": -251.46310424804688, |
| "loss": 0.005312850698828697, |
| "mean_token_accuracy": 0.6195648014545441, |
| "num_tokens": 112297.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.948453530203551, |
| "rewards/margins": 7.2391074895858765, |
| "rewards/rejected": -6.290653884410858, |
| "step": 17 |
| }, |
| { |
| "entropy": 1.1938088834285736, |
| "epoch": 0.9056603773584906, |
| "grad_norm": 0.0283203125, |
| "learning_rate": 3.5833333333333335e-05, |
| "logits/chosen": -2.0970170230360243, |
| "logits/rejected": -1.6893045219197895, |
| "logps/chosen": -128.71086406707764, |
| "logps/rejected": -248.6879997253418, |
| "loss": 0.0016415835125371814, |
| "mean_token_accuracy": 0.6066833287477493, |
| "num_tokens": 118259.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6370860114693642, |
| "rewards/margins": 7.347302317619324, |
| "rewards/rejected": -6.710216283798218, |
| "step": 18 |
| }, |
| { |
| "entropy": 1.1347751468420029, |
| "epoch": 0.9559748427672956, |
| "grad_norm": 0.021240234375, |
| "learning_rate": 3.5e-05, |
| "logits/chosen": -2.174784405399738, |
| "logits/rejected": -1.8171304190186441, |
| "logps/chosen": -137.3084011077881, |
| "logps/rejected": -237.20912742614746, |
| "loss": 0.0013000022154301405, |
| "mean_token_accuracy": 0.6105500273406506, |
| "num_tokens": 123618.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.1855077669024467, |
| "rewards/margins": 7.6381800174713135, |
| "rewards/rejected": -6.452672183513641, |
| "step": 19 |
| }, |
| { |
| "entropy": 1.2229801756995065, |
| "epoch": 1.0, |
| "grad_norm": 0.034912109375, |
| "learning_rate": 3.4166666666666666e-05, |
| "logits/chosen": -1.9402861332224905, |
| "logits/rejected": -1.478508916635409, |
| "logps/chosen": -140.62159075055803, |
| "logps/rejected": -258.58056640625, |
| "loss": 0.0018078959546983242, |
| "mean_token_accuracy": 0.6002635615212577, |
| "num_tokens": 130000.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.18390404965196336, |
| "rewards/margins": 7.131651401519775, |
| "rewards/rejected": -7.31555564062936, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.1338201016187668, |
| "epoch": 1.050314465408805, |
| "grad_norm": 0.00341796875, |
| "learning_rate": 3.3333333333333335e-05, |
| "logits/chosen": -2.2381289981693855, |
| "logits/rejected": -1.7685196120349025, |
| "logps/chosen": -132.71670532226562, |
| "logps/rejected": -270.32165908813477, |
| "loss": 0.00017723625933285803, |
| "mean_token_accuracy": 0.640245221555233, |
| "num_tokens": 135799.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.0271762115880847, |
| "rewards/margins": 9.069988369941711, |
| "rewards/rejected": -8.04281210899353, |
| "step": 21 |
| }, |
| { |
| "entropy": 1.184071958065033, |
| "epoch": 1.10062893081761, |
| "grad_norm": 0.06201171875, |
| "learning_rate": 3.2500000000000004e-05, |
| "logits/chosen": -2.2030959691219643, |
| "logits/rejected": -1.549622772314418, |
| "logps/chosen": -137.2278184890747, |
| "logps/rejected": -278.6941947937012, |
| "loss": 0.002708145882934332, |
| "mean_token_accuracy": 0.5984649807214737, |
| "num_tokens": 142686.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.10265790205448866, |
| "rewards/margins": 8.331138014793396, |
| "rewards/rejected": -8.228479981422424, |
| "step": 22 |
| }, |
| { |
| "entropy": 1.1163400262594223, |
| "epoch": 1.150943396226415, |
| "grad_norm": 0.010498046875, |
| "learning_rate": 3.1666666666666666e-05, |
| "logits/chosen": -2.2692944342560915, |
| "logits/rejected": -1.9815728330371412, |
| "logps/chosen": -121.65661811828613, |
| "logps/rejected": -251.65609550476074, |
| "loss": 0.0006153068970888853, |
| "mean_token_accuracy": 0.5803752839565277, |
| "num_tokens": 148888.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.6473507843911648, |
| "rewards/margins": 8.257275819778442, |
| "rewards/rejected": -7.609925091266632, |
| "step": 23 |
| }, |
| { |
| "entropy": 1.2022811472415924, |
| "epoch": 1.20125786163522, |
| "grad_norm": 0.0196533203125, |
| "learning_rate": 3.0833333333333335e-05, |
| "logits/chosen": -2.0684997205168574, |
| "logits/rejected": -1.5187979793505884, |
| "logps/chosen": -142.0221881866455, |
| "logps/rejected": -288.0162467956543, |
| "loss": 0.0010658408282324672, |
| "mean_token_accuracy": 0.6041584983468056, |
| "num_tokens": 155066.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.3129555657505989, |
| "rewards/margins": 9.287788093090057, |
| "rewards/rejected": -8.974832653999329, |
| "step": 24 |
| }, |
| { |
| "entropy": 1.0715540498495102, |
| "epoch": 1.251572327044025, |
| "grad_norm": 0.0152587890625, |
| "learning_rate": 3e-05, |
| "logits/chosen": -2.253688589577229, |
| "logits/rejected": -1.709912522981535, |
| "logps/chosen": -124.24794864654541, |
| "logps/rejected": -268.31462478637695, |
| "loss": 0.0007783680921420455, |
| "mean_token_accuracy": 0.6487728357315063, |
| "num_tokens": 161896.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5254653260344639, |
| "rewards/margins": 8.636412143707275, |
| "rewards/rejected": -8.110946834087372, |
| "step": 25 |
| }, |
| { |
| "entropy": 1.1292345225811005, |
| "epoch": 1.3018867924528301, |
| "grad_norm": 0.01611328125, |
| "learning_rate": 2.916666666666667e-05, |
| "logits/chosen": -2.03905456801847, |
| "logits/rejected": -1.653255032290945, |
| "logps/chosen": -121.39779376983643, |
| "logps/rejected": -255.01618576049805, |
| "loss": 0.0008846853161230683, |
| "mean_token_accuracy": 0.622931219637394, |
| "num_tokens": 168755.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.2767142332158983, |
| "rewards/margins": 8.121429800987244, |
| "rewards/rejected": -7.844715654850006, |
| "step": 26 |
| }, |
| { |
| "entropy": 1.107276238501072, |
| "epoch": 1.3522012578616351, |
| "grad_norm": 0.0076904296875, |
| "learning_rate": 2.8333333333333335e-05, |
| "logits/chosen": -2.286326659906524, |
| "logits/rejected": -1.9377510490705079, |
| "logps/chosen": -144.30573749542236, |
| "logps/rejected": -272.37299728393555, |
| "loss": 0.00033011281630024314, |
| "mean_token_accuracy": 0.619342528283596, |
| "num_tokens": 175381.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.9377570524811745, |
| "rewards/margins": 9.561910688877106, |
| "rewards/rejected": -8.6241534948349, |
| "step": 27 |
| }, |
| { |
| "entropy": 1.157460778951645, |
| "epoch": 1.4025157232704402, |
| "grad_norm": 0.0047607421875, |
| "learning_rate": 2.7500000000000004e-05, |
| "logits/chosen": -2.1326494507814266, |
| "logits/rejected": -1.8268033175884986, |
| "logps/chosen": -125.0985517501831, |
| "logps/rejected": -274.5887870788574, |
| "loss": 0.0002390609442954883, |
| "mean_token_accuracy": 0.620079979300499, |
| "num_tokens": 181771.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8465396109968424, |
| "rewards/margins": 9.534583747386932, |
| "rewards/rejected": -8.688044130802155, |
| "step": 28 |
| }, |
| { |
| "entropy": 1.1201048269867897, |
| "epoch": 1.4528301886792452, |
| "grad_norm": 0.0093994140625, |
| "learning_rate": 2.6666666666666667e-05, |
| "logits/chosen": -2.311015681209517, |
| "logits/rejected": -2.0115759556684885, |
| "logps/chosen": -140.10973072052002, |
| "logps/rejected": -260.4351615905762, |
| "loss": 0.0005316605675034225, |
| "mean_token_accuracy": 0.6181787773966789, |
| "num_tokens": 188644.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.1467662937939167, |
| "rewards/margins": 8.486627459526062, |
| "rewards/rejected": -8.633393704891205, |
| "step": 29 |
| }, |
| { |
| "entropy": 1.1074048355221748, |
| "epoch": 1.5031446540880502, |
| "grad_norm": 0.024658203125, |
| "learning_rate": 2.5833333333333336e-05, |
| "logits/chosen": -2.2242952052635343, |
| "logits/rejected": -1.9084164743872978, |
| "logps/chosen": -120.53641414642334, |
| "logps/rejected": -248.61384201049805, |
| "loss": 0.0015231292927637696, |
| "mean_token_accuracy": 0.6136712729930878, |
| "num_tokens": 196391.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.0413979523582384, |
| "rewards/margins": 8.107804000377655, |
| "rewards/rejected": -8.149202108383179, |
| "step": 30 |
| }, |
| { |
| "entropy": 1.0835556164383888, |
| "epoch": 1.5534591194968552, |
| "grad_norm": 0.0308837890625, |
| "learning_rate": 2.5e-05, |
| "logits/chosen": -2.2358070307390454, |
| "logits/rejected": -1.8015257262782527, |
| "logps/chosen": -160.49627590179443, |
| "logps/rejected": -286.36713790893555, |
| "loss": 0.0012202183715999126, |
| "mean_token_accuracy": 0.6140667796134949, |
| "num_tokens": 203518.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.12220276962034404, |
| "rewards/margins": 8.978903114795685, |
| "rewards/rejected": -9.10110592842102, |
| "step": 31 |
| }, |
| { |
| "entropy": 1.1327729299664497, |
| "epoch": 1.6037735849056602, |
| "grad_norm": 0.0078125, |
| "learning_rate": 2.4166666666666667e-05, |
| "logits/chosen": -2.222914372779498, |
| "logits/rejected": -1.8115026450741711, |
| "logps/chosen": -143.74180603027344, |
| "logps/rejected": -296.8906936645508, |
| "loss": 0.0003141126944683492, |
| "mean_token_accuracy": 0.6098875515162945, |
| "num_tokens": 209850.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.1798817589879036, |
| "rewards/margins": 10.448372840881348, |
| "rewards/rejected": -10.268491089344025, |
| "step": 32 |
| }, |
| { |
| "entropy": 1.125333271920681, |
| "epoch": 1.6540880503144653, |
| "grad_norm": 0.017578125, |
| "learning_rate": 2.3333333333333336e-05, |
| "logits/chosen": -2.4135272067915143, |
| "logits/rejected": -1.7676647261719525, |
| "logps/chosen": -125.86479663848877, |
| "logps/rejected": -266.06410026550293, |
| "loss": 0.0005237428122200072, |
| "mean_token_accuracy": 0.6229566335678101, |
| "num_tokens": 215637.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7084813113324344, |
| "rewards/margins": 9.689094841480255, |
| "rewards/rejected": -8.980613589286804, |
| "step": 33 |
| }, |
| { |
| "entropy": 1.058401882648468, |
| "epoch": 1.7044025157232703, |
| "grad_norm": 0.007354736328125, |
| "learning_rate": 2.25e-05, |
| "logits/chosen": -2.46378613488542, |
| "logits/rejected": -2.1071431291236653, |
| "logps/chosen": -125.6760368347168, |
| "logps/rejected": -258.0559844970703, |
| "loss": 0.00038854233571328223, |
| "mean_token_accuracy": 0.6303330659866333, |
| "num_tokens": 221051.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.3429398629814386, |
| "rewards/margins": 10.128098785877228, |
| "rewards/rejected": -8.785158812999725, |
| "step": 34 |
| }, |
| { |
| "entropy": 1.1321996748447418, |
| "epoch": 1.7547169811320755, |
| "grad_norm": 0.0030670166015625, |
| "learning_rate": 2.1666666666666667e-05, |
| "logits/chosen": -2.3747636222967063, |
| "logits/rejected": -2.115993388323722, |
| "logps/chosen": -131.67159175872803, |
| "logps/rejected": -276.36437797546387, |
| "loss": 0.00012030061770929024, |
| "mean_token_accuracy": 0.6251056790351868, |
| "num_tokens": 226426.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8489564098417759, |
| "rewards/margins": 10.621768474578857, |
| "rewards/rejected": -9.772812008857727, |
| "step": 35 |
| }, |
| { |
| "entropy": 1.116398274898529, |
| "epoch": 1.8050314465408805, |
| "grad_norm": 0.0181884765625, |
| "learning_rate": 2.0833333333333336e-05, |
| "logits/chosen": -2.4568723077785557, |
| "logits/rejected": -1.9738618373268513, |
| "logps/chosen": -117.32646560668945, |
| "logps/rejected": -272.7527599334717, |
| "loss": 0.0008142158621922135, |
| "mean_token_accuracy": 0.6250453069806099, |
| "num_tokens": 233419.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.3417433723807335, |
| "rewards/margins": 9.766010344028473, |
| "rewards/rejected": -9.424266993999481, |
| "step": 36 |
| }, |
| { |
| "entropy": 1.0965562090277672, |
| "epoch": 1.8553459119496856, |
| "grad_norm": 0.0027313232421875, |
| "learning_rate": 2e-05, |
| "logits/chosen": -2.3551532120785383, |
| "logits/rejected": -1.9812547363037738, |
| "logps/chosen": -144.9142713546753, |
| "logps/rejected": -296.3636951446533, |
| "loss": 0.00013589797890745103, |
| "mean_token_accuracy": 0.5917238146066666, |
| "num_tokens": 240673.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.10223913192749023, |
| "rewards/margins": 10.127011775970459, |
| "rewards/rejected": -10.22925090789795, |
| "step": 37 |
| }, |
| { |
| "entropy": 1.088953472673893, |
| "epoch": 1.9056603773584906, |
| "grad_norm": 0.0032806396484375, |
| "learning_rate": 1.9166666666666667e-05, |
| "logits/chosen": -2.326897647347544, |
| "logits/rejected": -2.0923411183004155, |
| "logps/chosen": -141.62347412109375, |
| "logps/rejected": -285.8790645599365, |
| "loss": 0.0001322527095908299, |
| "mean_token_accuracy": 0.6075941771268845, |
| "num_tokens": 245974.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7096749469637871, |
| "rewards/margins": 10.769983649253845, |
| "rewards/rejected": -10.060308814048767, |
| "step": 38 |
| }, |
| { |
| "entropy": 1.1708777844905853, |
| "epoch": 1.9559748427672956, |
| "grad_norm": 0.003997802734375, |
| "learning_rate": 1.8333333333333333e-05, |
| "logits/chosen": -2.1918049696005726, |
| "logits/rejected": -1.7380488443097255, |
| "logps/chosen": -127.21480464935303, |
| "logps/rejected": -273.00823402404785, |
| "loss": 0.00020453613251447678, |
| "mean_token_accuracy": 0.6081509068608284, |
| "num_tokens": 253184.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.09920912818051875, |
| "rewards/margins": 9.214752078056335, |
| "rewards/rejected": -9.313961207866669, |
| "step": 39 |
| }, |
| { |
| "entropy": 1.1617241672107153, |
| "epoch": 2.0, |
| "grad_norm": 0.0191650390625, |
| "learning_rate": 1.75e-05, |
| "logits/chosen": -2.2749118295696342, |
| "logits/rejected": -1.7584887991939784, |
| "logps/chosen": -154.0042724609375, |
| "logps/rejected": -292.14568001883373, |
| "loss": 0.0009170390549115837, |
| "mean_token_accuracy": 0.5644707764898028, |
| "num_tokens": 260000.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.44687261964593616, |
| "rewards/margins": 9.13522618157523, |
| "rewards/rejected": -9.582098756517683, |
| "step": 40 |
| }, |
| { |
| "entropy": 1.1219489574432373, |
| "epoch": 2.050314465408805, |
| "grad_norm": 0.00396728515625, |
| "learning_rate": 1.6666666666666667e-05, |
| "logits/chosen": -2.3933433594531426, |
| "logits/rejected": -1.8782996338947386, |
| "logps/chosen": -121.1051197052002, |
| "logps/rejected": -273.1605224609375, |
| "loss": 0.00018580301548354328, |
| "mean_token_accuracy": 0.6062684953212738, |
| "num_tokens": 266060.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5098631188739091, |
| "rewards/margins": 9.81256103515625, |
| "rewards/rejected": -9.302697837352753, |
| "step": 41 |
| }, |
| { |
| "entropy": 1.142582356929779, |
| "epoch": 2.10062893081761, |
| "grad_norm": 0.00506591796875, |
| "learning_rate": 1.5833333333333333e-05, |
| "logits/chosen": -2.3932424547453, |
| "logits/rejected": -2.034844354918984, |
| "logps/chosen": -132.19951725006104, |
| "logps/rejected": -273.3875484466553, |
| "loss": 0.00027762039098888636, |
| "mean_token_accuracy": 0.6254889070987701, |
| "num_tokens": 272769.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.027547355741262436, |
| "rewards/margins": 9.588396728038788, |
| "rewards/rejected": -9.560849368572235, |
| "step": 42 |
| }, |
| { |
| "entropy": 1.0831276252865791, |
| "epoch": 2.150943396226415, |
| "grad_norm": 0.0201416015625, |
| "learning_rate": 1.5e-05, |
| "logits/chosen": -2.3213323581687466, |
| "logits/rejected": -1.9188541617869985, |
| "logps/chosen": -140.96678638458252, |
| "logps/rejected": -298.01341819763184, |
| "loss": 0.0006403317674994469, |
| "mean_token_accuracy": 0.6212100088596344, |
| "num_tokens": 279603.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.13022289611399174, |
| "rewards/margins": 10.482895195484161, |
| "rewards/rejected": -10.61311811208725, |
| "step": 43 |
| }, |
| { |
| "entropy": 1.1165131032466888, |
| "epoch": 2.20125786163522, |
| "grad_norm": 0.004119873046875, |
| "learning_rate": 1.4166666666666668e-05, |
| "logits/chosen": -2.325539811977018, |
| "logits/rejected": -1.819132333499079, |
| "logps/chosen": -137.1863489151001, |
| "logps/rejected": -280.5503120422363, |
| "loss": 0.00018718022329267114, |
| "mean_token_accuracy": 0.6205575913190842, |
| "num_tokens": 286436.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.0954840648919344, |
| "rewards/margins": 9.663193047046661, |
| "rewards/rejected": -9.567708969116211, |
| "step": 44 |
| }, |
| { |
| "entropy": 1.0798147842288017, |
| "epoch": 2.251572327044025, |
| "grad_norm": 0.006195068359375, |
| "learning_rate": 1.3333333333333333e-05, |
| "logits/chosen": -2.462720523269614, |
| "logits/rejected": -1.9150426029399608, |
| "logps/chosen": -137.90959072113037, |
| "logps/rejected": -299.62617683410645, |
| "loss": 0.00032327763619832695, |
| "mean_token_accuracy": 0.6055086851119995, |
| "num_tokens": 294223.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.0803464986383915, |
| "rewards/margins": 10.386663496494293, |
| "rewards/rejected": -10.467009961605072, |
| "step": 45 |
| }, |
| { |
| "entropy": 1.0406965985894203, |
| "epoch": 2.30188679245283, |
| "grad_norm": 0.00360107421875, |
| "learning_rate": 1.25e-05, |
| "logits/chosen": -2.351859813790029, |
| "logits/rejected": -2.079019230872164, |
| "logps/chosen": -114.20672798156738, |
| "logps/rejected": -270.77587890625, |
| "loss": 0.0001596831134520471, |
| "mean_token_accuracy": 0.6743896827101707, |
| "num_tokens": 299634.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.2796257368754596, |
| "rewards/margins": 10.65468156337738, |
| "rewards/rejected": -9.375055730342865, |
| "step": 46 |
| }, |
| { |
| "entropy": 1.1319249421358109, |
| "epoch": 2.352201257861635, |
| "grad_norm": 0.00323486328125, |
| "learning_rate": 1.1666666666666668e-05, |
| "logits/chosen": -2.3348495432588066, |
| "logits/rejected": -1.8859447778318736, |
| "logps/chosen": -162.34913635253906, |
| "logps/rejected": -305.2253665924072, |
| "loss": 0.00012244780373293906, |
| "mean_token_accuracy": 0.5992361158132553, |
| "num_tokens": 305920.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.0735319098457694, |
| "rewards/margins": 10.709958910942078, |
| "rewards/rejected": -10.783490896224976, |
| "step": 47 |
| }, |
| { |
| "entropy": 1.1012349873781204, |
| "epoch": 2.40251572327044, |
| "grad_norm": 0.00555419921875, |
| "learning_rate": 1.0833333333333334e-05, |
| "logits/chosen": -2.376434497997167, |
| "logits/rejected": -1.9649504169742804, |
| "logps/chosen": -134.29817581176758, |
| "logps/rejected": -269.4762592315674, |
| "loss": 0.00029120981344021857, |
| "mean_token_accuracy": 0.615195669233799, |
| "num_tokens": 313346.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.04450349509716034, |
| "rewards/margins": 8.847583532333374, |
| "rewards/rejected": -8.803080201148987, |
| "step": 48 |
| }, |
| { |
| "entropy": 1.0822330936789513, |
| "epoch": 2.452830188679245, |
| "grad_norm": 0.004486083984375, |
| "learning_rate": 1e-05, |
| "logits/chosen": -2.4181823111443506, |
| "logits/rejected": -1.9432117626083543, |
| "logps/chosen": -139.22668647766113, |
| "logps/rejected": -286.98141860961914, |
| "loss": 0.00024342790129594505, |
| "mean_token_accuracy": 0.5807452276349068, |
| "num_tokens": 320662.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.20274513261392713, |
| "rewards/margins": 9.553876638412476, |
| "rewards/rejected": -9.756621778011322, |
| "step": 49 |
| }, |
| { |
| "entropy": 1.1255912110209465, |
| "epoch": 2.50314465408805, |
| "grad_norm": 0.01043701171875, |
| "learning_rate": 9.166666666666666e-06, |
| "logits/chosen": -2.390428515839434, |
| "logits/rejected": -2.070867890248319, |
| "logps/chosen": -152.57851219177246, |
| "logps/rejected": -278.1038990020752, |
| "loss": 0.0005284082726575434, |
| "mean_token_accuracy": 0.5914618484675884, |
| "num_tokens": 326611.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.24946272000670433, |
| "rewards/margins": 9.3264040350914, |
| "rewards/rejected": -9.076941430568695, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.10995664447546, |
| "epoch": 2.5534591194968552, |
| "grad_norm": 0.005950927734375, |
| "learning_rate": 8.333333333333334e-06, |
| "logits/chosen": -2.3087954809493545, |
| "logits/rejected": -1.9792274410316972, |
| "logps/chosen": -116.59206485748291, |
| "logps/rejected": -267.238338470459, |
| "loss": 0.00025631688185967505, |
| "mean_token_accuracy": 0.625831313431263, |
| "num_tokens": 332814.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.8696157680824399, |
| "rewards/margins": 10.124218106269836, |
| "rewards/rejected": -9.254602372646332, |
| "step": 51 |
| }, |
| { |
| "entropy": 1.117017239332199, |
| "epoch": 2.6037735849056602, |
| "grad_norm": 0.00152587890625, |
| "learning_rate": 7.5e-06, |
| "logits/chosen": -2.4137662915261817, |
| "logits/rejected": -1.9523699949115683, |
| "logps/chosen": -120.80679893493652, |
| "logps/rejected": -280.64561653137207, |
| "loss": 7.924844976514578e-05, |
| "mean_token_accuracy": 0.6106409505009651, |
| "num_tokens": 339547.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.790530975908041, |
| "rewards/margins": 10.681683659553528, |
| "rewards/rejected": -9.891152799129486, |
| "step": 52 |
| }, |
| { |
| "entropy": 1.1054791137576103, |
| "epoch": 2.6540880503144653, |
| "grad_norm": 0.0164794921875, |
| "learning_rate": 6.666666666666667e-06, |
| "logits/chosen": -2.273507778205582, |
| "logits/rejected": -1.9109036313937824, |
| "logps/chosen": -115.94961833953857, |
| "logps/rejected": -269.0294952392578, |
| "loss": 0.0007658082176931202, |
| "mean_token_accuracy": 0.5990233793854713, |
| "num_tokens": 346647.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.25752640701830387, |
| "rewards/margins": 9.45040088891983, |
| "rewards/rejected": -9.192874610424042, |
| "step": 53 |
| }, |
| { |
| "entropy": 1.103651024401188, |
| "epoch": 2.7044025157232703, |
| "grad_norm": 0.002593994140625, |
| "learning_rate": 5.833333333333334e-06, |
| "logits/chosen": -2.333249501998797, |
| "logits/rejected": -1.8058473440003249, |
| "logps/chosen": -152.4753885269165, |
| "logps/rejected": -301.55738639831543, |
| "loss": 0.00010592853504931554, |
| "mean_token_accuracy": 0.6078185737133026, |
| "num_tokens": 353155.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.3159999940544367, |
| "rewards/margins": 10.630276560783386, |
| "rewards/rejected": -10.314276397228241, |
| "step": 54 |
| }, |
| { |
| "entropy": 1.160094790160656, |
| "epoch": 2.7547169811320753, |
| "grad_norm": 0.0015716552734375, |
| "learning_rate": 5e-06, |
| "logits/chosen": -2.3308505618468462, |
| "logits/rejected": -1.8965502590056609, |
| "logps/chosen": -141.07257843017578, |
| "logps/rejected": -296.07042503356934, |
| "loss": 6.342934648273513e-05, |
| "mean_token_accuracy": 0.6171156838536263, |
| "num_tokens": 358538.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.7141566202044487, |
| "rewards/margins": 10.787114977836609, |
| "rewards/rejected": -10.072958290576935, |
| "step": 55 |
| }, |
| { |
| "entropy": 1.1058026999235153, |
| "epoch": 2.8050314465408803, |
| "grad_norm": 0.030517578125, |
| "learning_rate": 4.166666666666667e-06, |
| "logits/chosen": -2.322696784138965, |
| "logits/rejected": -1.894479697226797, |
| "logps/chosen": -121.85036373138428, |
| "logps/rejected": -254.1634178161621, |
| "loss": 0.00172812445089221, |
| "mean_token_accuracy": 0.6225817948579788, |
| "num_tokens": 364730.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.10639754496514797, |
| "rewards/margins": 8.579113006591797, |
| "rewards/rejected": -8.472715437412262, |
| "step": 56 |
| }, |
| { |
| "entropy": 1.102943792939186, |
| "epoch": 2.8553459119496853, |
| "grad_norm": 0.00872802734375, |
| "learning_rate": 3.3333333333333333e-06, |
| "logits/chosen": -2.173708624672683, |
| "logits/rejected": -1.970646926930764, |
| "logps/chosen": -142.89206504821777, |
| "logps/rejected": -268.407958984375, |
| "loss": 0.00041806488297879696, |
| "mean_token_accuracy": 0.5970962047576904, |
| "num_tokens": 371209.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.27814904414117336, |
| "rewards/margins": 9.40092819929123, |
| "rewards/rejected": -9.122779071331024, |
| "step": 57 |
| }, |
| { |
| "entropy": 1.1477283239364624, |
| "epoch": 2.9056603773584904, |
| "grad_norm": 0.00469970703125, |
| "learning_rate": 2.5e-06, |
| "logits/chosen": -2.293317013131777, |
| "logits/rejected": -1.8812048808544868, |
| "logps/chosen": -137.3569221496582, |
| "logps/rejected": -291.4598960876465, |
| "loss": 0.00018517834541853517, |
| "mean_token_accuracy": 0.6110436543822289, |
| "num_tokens": 378077.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.33568347059190273, |
| "rewards/margins": 10.355388283729553, |
| "rewards/rejected": -10.01970487833023, |
| "step": 58 |
| }, |
| { |
| "entropy": 1.0955762341618538, |
| "epoch": 2.9559748427672954, |
| "grad_norm": 0.00193023681640625, |
| "learning_rate": 1.6666666666666667e-06, |
| "logits/chosen": -2.3105184309249727, |
| "logits/rejected": -2.0821060341980577, |
| "logps/chosen": -141.80660915374756, |
| "logps/rejected": -279.77105140686035, |
| "loss": 8.936067024478689e-05, |
| "mean_token_accuracy": 0.6105019077658653, |
| "num_tokens": 384115.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 0.5417802967131138, |
| "rewards/margins": 10.488943040370941, |
| "rewards/rejected": -9.947162747383118, |
| "step": 59 |
| }, |
| { |
| "entropy": 1.0900011318070548, |
| "epoch": 3.0, |
| "grad_norm": 0.024169921875, |
| "learning_rate": 8.333333333333333e-07, |
| "logits/chosen": -2.332351866206725, |
| "logits/rejected": -1.8031474708962778, |
| "logps/chosen": -138.6800994873047, |
| "logps/rejected": -265.439217703683, |
| "loss": 0.0013465541414916515, |
| "mean_token_accuracy": 0.6026440688541957, |
| "num_tokens": 390000.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2070258174623762, |
| "rewards/margins": 8.572459425245013, |
| "rewards/rejected": -8.77948522567749, |
| "step": 60 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 60, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 60, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 3189182437748736.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
