| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 500, |
| "global_step": 1467, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 2.163015019595623, |
| "epoch": 0.01704448610874382, |
| "grad_norm": 0.050537109375, |
| "learning_rate": 0.00010666666666666667, |
| "loss": 2.4524998474121094, |
| "mean_token_accuracy": 0.5153323707729578, |
| "num_tokens": 132137.0, |
| "step": 25 |
| }, |
| { |
| "entropy": 2.2185084584355352, |
| "epoch": 0.03408897221748764, |
| "grad_norm": 0.068359375, |
| "learning_rate": 0.000199996095299633, |
| "loss": 2.173491668701172, |
| "mean_token_accuracy": 0.5537562420964242, |
| "num_tokens": 261404.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.9352111649513244, |
| "epoch": 0.051133458326231465, |
| "grad_norm": 0.1376953125, |
| "learning_rate": 0.0001997948280489004, |
| "loss": 1.8705439758300781, |
| "mean_token_accuracy": 0.5959619037806988, |
| "num_tokens": 396323.0, |
| "step": 75 |
| }, |
| { |
| "entropy": 1.76205413736403, |
| "epoch": 0.06817794443497528, |
| "grad_norm": 0.09423828125, |
| "learning_rate": 0.00019928920737019733, |
| "loss": 1.6511923217773437, |
| "mean_token_accuracy": 0.6337622724473476, |
| "num_tokens": 527358.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 1.4949746784381568, |
| "epoch": 0.08522243054371911, |
| "grad_norm": 0.0625, |
| "learning_rate": 0.00019848077530122083, |
| "loss": 1.3673600769042968, |
| "mean_token_accuracy": 0.6848376739770174, |
| "num_tokens": 655447.0, |
| "step": 125 |
| }, |
| { |
| "entropy": 1.4837670608237385, |
| "epoch": 0.10226691665246293, |
| "grad_norm": 0.08837890625, |
| "learning_rate": 0.0001973719973913012, |
| "loss": 1.4141250610351563, |
| "mean_token_accuracy": 0.6836329951137304, |
| "num_tokens": 790490.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 1.3150090302992612, |
| "epoch": 0.11931140276120675, |
| "grad_norm": 0.091796875, |
| "learning_rate": 0.00019596625518199077, |
| "loss": 1.2282736968994141, |
| "mean_token_accuracy": 0.7172392677515745, |
| "num_tokens": 925069.0, |
| "step": 175 |
| }, |
| { |
| "entropy": 1.3966816508676856, |
| "epoch": 0.13635588886995056, |
| "grad_norm": 0.0712890625, |
| "learning_rate": 0.00019426783589406642, |
| "loss": 1.3173268127441407, |
| "mean_token_accuracy": 0.7017394271492958, |
| "num_tokens": 1052891.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 1.2435051177628338, |
| "epoch": 0.15340037497869438, |
| "grad_norm": 0.042236328125, |
| "learning_rate": 0.00019228191935240007, |
| "loss": 1.115916519165039, |
| "mean_token_accuracy": 0.7378023877739907, |
| "num_tokens": 1184065.0, |
| "step": 225 |
| }, |
| { |
| "entropy": 1.3924890440981834, |
| "epoch": 0.17044486108743823, |
| "grad_norm": 0.072265625, |
| "learning_rate": 0.00019001456218857208, |
| "loss": 1.288157958984375, |
| "mean_token_accuracy": 0.7004149560630322, |
| "num_tokens": 1313612.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 1.3593159637693315, |
| "epoch": 0.18748934719618204, |
| "grad_norm": 0.08740234375, |
| "learning_rate": 0.00018747267936940722, |
| "loss": 1.2840399169921874, |
| "mean_token_accuracy": 0.7129683202505112, |
| "num_tokens": 1444287.0, |
| "step": 275 |
| }, |
| { |
| "entropy": 1.2371155321784317, |
| "epoch": 0.20453383330492586, |
| "grad_norm": 0.09130859375, |
| "learning_rate": 0.00018466402310776708, |
| "loss": 1.1687215423583985, |
| "mean_token_accuracy": 0.73335683144629, |
| "num_tokens": 1577307.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 1.2478438840154558, |
| "epoch": 0.22157831941366968, |
| "grad_norm": 0.06005859375, |
| "learning_rate": 0.00018159715921991612, |
| "loss": 1.20022216796875, |
| "mean_token_accuracy": 0.732666322439909, |
| "num_tokens": 1711348.0, |
| "step": 325 |
| }, |
| { |
| "entropy": 1.1931916671898215, |
| "epoch": 0.2386228055224135, |
| "grad_norm": 0.103515625, |
| "learning_rate": 0.00017828144100156754, |
| "loss": 1.082593002319336, |
| "mean_token_accuracy": 0.7456949570775032, |
| "num_tokens": 1842011.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 1.1265690931119026, |
| "epoch": 0.2556672916311573, |
| "grad_norm": 0.0888671875, |
| "learning_rate": 0.00017472698070228047, |
| "loss": 1.0630721282958984, |
| "mean_token_accuracy": 0.7553409574180842, |
| "num_tokens": 1976056.0, |
| "step": 375 |
| }, |
| { |
| "entropy": 1.1145534914545714, |
| "epoch": 0.27271177773990113, |
| "grad_norm": 0.06787109375, |
| "learning_rate": 0.00017094461868520622, |
| "loss": 1.0225981903076171, |
| "mean_token_accuracy": 0.7580388783663511, |
| "num_tokens": 2109389.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 1.25198206695728, |
| "epoch": 0.28975626384864495, |
| "grad_norm": 0.048828125, |
| "learning_rate": 0.00016694589036623988, |
| "loss": 1.194885482788086, |
| "mean_token_accuracy": 0.7303700438886881, |
| "num_tokens": 2240117.0, |
| "step": 425 |
| }, |
| { |
| "entropy": 1.1865128787327557, |
| "epoch": 0.30680074995738876, |
| "grad_norm": 0.0556640625, |
| "learning_rate": 0.0001627429910334058, |
| "loss": 1.118544158935547, |
| "mean_token_accuracy": 0.7400260034948587, |
| "num_tokens": 2364080.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 1.274649427868426, |
| "epoch": 0.3238452360661326, |
| "grad_norm": 0.06591796875, |
| "learning_rate": 0.00015834873865377077, |
| "loss": 1.1928933715820313, |
| "mean_token_accuracy": 0.7288886807858944, |
| "num_tokens": 2497166.0, |
| "step": 475 |
| }, |
| { |
| "entropy": 1.1076703028334305, |
| "epoch": 0.34088972217487645, |
| "grad_norm": 0.0693359375, |
| "learning_rate": 0.00015377653478131556, |
| "loss": 1.0115528869628907, |
| "mean_token_accuracy": 0.7560868422687054, |
| "num_tokens": 2632559.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 1.1313659130921587, |
| "epoch": 0.35793420828362027, |
| "grad_norm": 0.08935546875, |
| "learning_rate": 0.00014904032368498856, |
| "loss": 1.0595830535888673, |
| "mean_token_accuracy": 0.756401935890317, |
| "num_tokens": 2764478.0, |
| "step": 525 |
| }, |
| { |
| "entropy": 1.0654471471440048, |
| "epoch": 0.3749786943923641, |
| "grad_norm": 0.0771484375, |
| "learning_rate": 0.0001441545498215912, |
| "loss": 1.0119430541992187, |
| "mean_token_accuracy": 0.768122655749321, |
| "num_tokens": 2901372.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 1.2444889663858338, |
| "epoch": 0.3920231805011079, |
| "grad_norm": 0.0712890625, |
| "learning_rate": 0.00013913411378319504, |
| "loss": 1.2215270233154296, |
| "mean_token_accuracy": 0.729380588978529, |
| "num_tokens": 3031999.0, |
| "step": 575 |
| }, |
| { |
| "entropy": 1.0762281655799597, |
| "epoch": 0.4090676666098517, |
| "grad_norm": 0.08740234375, |
| "learning_rate": 0.0001339943268534414, |
| "loss": 0.9998902893066406, |
| "mean_token_accuracy": 0.768867435157299, |
| "num_tokens": 3164064.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 1.0626403710339218, |
| "epoch": 0.42611215271859554, |
| "grad_norm": 0.134765625, |
| "learning_rate": 0.00012875086431131716, |
| "loss": 0.9624679565429688, |
| "mean_token_accuracy": 0.771867539063096, |
| "num_tokens": 3288692.0, |
| "step": 625 |
| }, |
| { |
| "entropy": 0.9459870021557435, |
| "epoch": 0.44315663882733936, |
| "grad_norm": 0.072265625, |
| "learning_rate": 0.00012341971762482043, |
| "loss": 0.9064933776855468, |
| "mean_token_accuracy": 0.7910215198248625, |
| "num_tokens": 3418627.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 1.045862675276585, |
| "epoch": 0.4602011249360832, |
| "grad_norm": 0.06494140625, |
| "learning_rate": 0.00011801714568031615, |
| "loss": 0.9381060791015625, |
| "mean_token_accuracy": 0.770117818787694, |
| "num_tokens": 3545140.0, |
| "step": 675 |
| }, |
| { |
| "entropy": 0.9516787563310936, |
| "epoch": 0.477245611044827, |
| "grad_norm": 0.07666015625, |
| "learning_rate": 0.00011255962519632081, |
| "loss": 0.8808039093017578, |
| "mean_token_accuracy": 0.7909089859575034, |
| "num_tokens": 3679692.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.9953352324664593, |
| "epoch": 0.4942900971535708, |
| "grad_norm": 0.051513671875, |
| "learning_rate": 0.00010706380047294496, |
| "loss": 0.9232154846191406, |
| "mean_token_accuracy": 0.7844866032898427, |
| "num_tokens": 3810488.0, |
| "step": 725 |
| }, |
| { |
| "entropy": 0.92748022117652, |
| "epoch": 0.5113345832623146, |
| "grad_norm": 0.07421875, |
| "learning_rate": 0.00010154643263024644, |
| "loss": 0.8504108428955078, |
| "mean_token_accuracy": 0.7934545055776835, |
| "num_tokens": 3937101.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.9793470423854888, |
| "epoch": 0.5283790693710585, |
| "grad_norm": 0.0771484375, |
| "learning_rate": 9.602434849030745e-05, |
| "loss": 0.8816374969482422, |
| "mean_token_accuracy": 0.786624568477273, |
| "num_tokens": 4069539.0, |
| "step": 775 |
| }, |
| { |
| "entropy": 0.9201436020294205, |
| "epoch": 0.5454235554798023, |
| "grad_norm": 0.06005859375, |
| "learning_rate": 9.051438925893384e-05, |
| "loss": 0.8456056976318359, |
| "mean_token_accuracy": 0.7975382874906063, |
| "num_tokens": 4196197.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.8780001322366298, |
| "epoch": 0.5624680415885461, |
| "grad_norm": 0.06787109375, |
| "learning_rate": 8.503335916348721e-05, |
| "loss": 0.7664035034179687, |
| "mean_token_accuracy": 0.8094721902906895, |
| "num_tokens": 4328551.0, |
| "step": 825 |
| }, |
| { |
| "entropy": 1.0318437330564485, |
| "epoch": 0.5795125276972899, |
| "grad_norm": 0.0625, |
| "learning_rate": 7.959797420349355e-05, |
| "loss": 0.9603571319580078, |
| "mean_token_accuracy": 0.7762538179755211, |
| "num_tokens": 4461738.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 1.001236357828602, |
| "epoch": 0.5965570138060338, |
| "grad_norm": 0.05322265625, |
| "learning_rate": 7.422481117032823e-05, |
| "loss": 0.9705011749267578, |
| "mean_token_accuracy": 0.7822855979949236, |
| "num_tokens": 4595951.0, |
| "step": 875 |
| }, |
| { |
| "entropy": 0.9190932653006166, |
| "epoch": 0.6136014999147775, |
| "grad_norm": 0.1259765625, |
| "learning_rate": 6.893025709145718e-05, |
| "loss": 0.7840510559082031, |
| "mean_token_accuracy": 0.7955148361623288, |
| "num_tokens": 4721963.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.9409526876406744, |
| "epoch": 0.6306459860235214, |
| "grad_norm": 0.07421875, |
| "learning_rate": 6.373045925341794e-05, |
| "loss": 0.8891078948974609, |
| "mean_token_accuracy": 0.797613599896431, |
| "num_tokens": 4857248.0, |
| "step": 925 |
| }, |
| { |
| "entropy": 0.9927360657276586, |
| "epoch": 0.6476904721322652, |
| "grad_norm": 0.05859375, |
| "learning_rate": 5.864127595596118e-05, |
| "loss": 0.9160787963867187, |
| "mean_token_accuracy": 0.7845662014186382, |
| "num_tokens": 4990977.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 1.0460572233749554, |
| "epoch": 0.664734958241009, |
| "grad_norm": 0.048583984375, |
| "learning_rate": 5.367822814754124e-05, |
| "loss": 1.0131810760498048, |
| "mean_token_accuracy": 0.7867030793055892, |
| "num_tokens": 5126029.0, |
| "step": 975 |
| }, |
| { |
| "entropy": 0.9399900265224278, |
| "epoch": 0.6817794443497529, |
| "grad_norm": 0.0810546875, |
| "learning_rate": 4.885645208965779e-05, |
| "loss": 0.8525303649902344, |
| "mean_token_accuracy": 0.7958442622423172, |
| "num_tokens": 5258088.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.9254711728263646, |
| "epoch": 0.6988239304584967, |
| "grad_norm": 0.06298828125, |
| "learning_rate": 4.419065319441306e-05, |
| "loss": 0.8452438354492188, |
| "mean_token_accuracy": 0.8007816936075688, |
| "num_tokens": 5389128.0, |
| "step": 1025 |
| }, |
| { |
| "entropy": 0.9583898686571046, |
| "epoch": 0.7158684165672405, |
| "grad_norm": 0.0615234375, |
| "learning_rate": 3.969506117606927e-05, |
| "loss": 0.910721435546875, |
| "mean_token_accuracy": 0.7937231123447418, |
| "num_tokens": 5518384.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.9213999844761566, |
| "epoch": 0.7329129026759843, |
| "grad_norm": 0.0625, |
| "learning_rate": 3.538338665338589e-05, |
| "loss": 0.8783349609375, |
| "mean_token_accuracy": 0.7985107507556677, |
| "num_tokens": 5649311.0, |
| "step": 1075 |
| }, |
| { |
| "entropy": 1.0327887642104179, |
| "epoch": 0.7499573887847282, |
| "grad_norm": 0.0810546875, |
| "learning_rate": 3.126877933508957e-05, |
| "loss": 0.9161809539794922, |
| "mean_token_accuracy": 0.7725442545861005, |
| "num_tokens": 5777733.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 1.0169112877128645, |
| "epoch": 0.7670018748934719, |
| "grad_norm": 0.0595703125, |
| "learning_rate": 2.7363787916003037e-05, |
| "loss": 0.9725022888183594, |
| "mean_token_accuracy": 0.777765890210867, |
| "num_tokens": 5909211.0, |
| "step": 1125 |
| }, |
| { |
| "entropy": 0.9282390323840082, |
| "epoch": 0.7840463610022158, |
| "grad_norm": 0.06298828125, |
| "learning_rate": 2.368032180614118e-05, |
| "loss": 0.8365528869628907, |
| "mean_token_accuracy": 0.7959657093882561, |
| "num_tokens": 6044115.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.9648981914808974, |
| "epoch": 0.8010908471109596, |
| "grad_norm": 0.05712890625, |
| "learning_rate": 2.0229614809492303e-05, |
| "loss": 0.8802178955078125, |
| "mean_token_accuracy": 0.7891545896232128, |
| "num_tokens": 6175533.0, |
| "step": 1175 |
| }, |
| { |
| "entropy": 0.8922164059896022, |
| "epoch": 0.8181353332197034, |
| "grad_norm": 0.05859375, |
| "learning_rate": 1.7022190863257183e-05, |
| "loss": 0.8290106201171875, |
| "mean_token_accuracy": 0.8064813084155321, |
| "num_tokens": 6310296.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.9672217888385057, |
| "epoch": 0.8351798193284472, |
| "grad_norm": 0.06884765625, |
| "learning_rate": 1.4067831942033904e-05, |
| "loss": 0.908281478881836, |
| "mean_token_accuracy": 0.7912823136150837, |
| "num_tokens": 6436503.0, |
| "step": 1225 |
| }, |
| { |
| "entropy": 1.0066828985884786, |
| "epoch": 0.8522243054371911, |
| "grad_norm": 0.06494140625, |
| "learning_rate": 1.137554822483381e-05, |
| "loss": 0.9817295074462891, |
| "mean_token_accuracy": 0.7799629791080952, |
| "num_tokens": 6572817.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.8913562398357316, |
| "epoch": 0.8692687915459348, |
| "grad_norm": 0.07421875, |
| "learning_rate": 8.953550615913397e-06, |
| "loss": 0.7918276977539063, |
| "mean_token_accuracy": 0.8021628592163325, |
| "num_tokens": 6701599.0, |
| "step": 1275 |
| }, |
| { |
| "entropy": 0.9468374169385061, |
| "epoch": 0.8863132776546787, |
| "grad_norm": 0.07958984375, |
| "learning_rate": 6.809225703227351e-06, |
| "loss": 0.8823848724365234, |
| "mean_token_accuracy": 0.7927402850985527, |
| "num_tokens": 6838746.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.9219346580654383, |
| "epoch": 0.9033577637634226, |
| "grad_norm": 0.0869140625, |
| "learning_rate": 4.949113230874847e-06, |
| "loss": 0.8668992614746094, |
| "mean_token_accuracy": 0.797563666626811, |
| "num_tokens": 6971712.0, |
| "step": 1325 |
| }, |
| { |
| "entropy": 0.984245891966857, |
| "epoch": 0.9204022498721663, |
| "grad_norm": 0.07666015625, |
| "learning_rate": 3.3788861542428553e-06, |
| "loss": 0.9503571319580079, |
| "mean_token_accuracy": 0.792408836260438, |
| "num_tokens": 7106279.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.913267534552142, |
| "epoch": 0.9374467359809102, |
| "grad_norm": 0.115234375, |
| "learning_rate": 2.1033333386744846e-06, |
| "loss": 0.8369984436035156, |
| "mean_token_accuracy": 0.803322741240263, |
| "num_tokens": 7238150.0, |
| "step": 1375 |
| }, |
| { |
| "entropy": 0.8992303860466927, |
| "epoch": 0.954491222089654, |
| "grad_norm": 0.064453125, |
| "learning_rate": 1.1263449544276938e-06, |
| "loss": 0.8289890289306641, |
| "mean_token_accuracy": 0.8079592730104923, |
| "num_tokens": 7374135.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.9052433096244932, |
| "epoch": 0.9715357081983979, |
| "grad_norm": 0.0498046875, |
| "learning_rate": 4.5090061246676475e-07, |
| "loss": 0.7991035461425782, |
| "mean_token_accuracy": 0.7997805786132812, |
| "num_tokens": 7508742.0, |
| "step": 1425 |
| }, |
| { |
| "entropy": 0.9894003225583583, |
| "epoch": 0.9885801943071416, |
| "grad_norm": 0.10595703125, |
| "learning_rate": 7.906027726981568e-08, |
| "loss": 0.9240727233886719, |
| "mean_token_accuracy": 0.786658907160163, |
| "num_tokens": 7649672.0, |
| "step": 1450 |
| } |
| ], |
| "logging_steps": 25, |
| "max_steps": 1467, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 3.3015596632508314e+17, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
