Upload tokenizer
1d4108a verified | { |
| "__af__": 128004, |
| "__am__": 128005, |
| "__ar__": 128006, |
| "__ast__": 128007, |
| "__az__": 128008, |
| "__ba__": 128009, |
| "__be__": 128010, |
| "__bg__": 128011, |
| "__bn__": 128012, |
| "__br__": 128013, |
| "__bs__": 128014, |
| "__ca__": 128015, |
| "__ceb__": 128016, |
| "__cs__": 128017, |
| "__cy__": 128018, |
| "__da__": 128019, |
| "__de__": 128020, |
| "__el__": 128021, |
| "__en__": 128022, |
| "__es__": 128023, |
| "__et__": 128024, |
| "__fa__": 128025, |
| "__ff__": 128026, |
| "__fi__": 128027, |
| "__fr__": 128028, |
| "__fy__": 128029, |
| "__ga__": 128030, |
| "__gd__": 128031, |
| "__gl__": 128032, |
| "__gu__": 128033, |
| "__ha__": 128034, |
| "__he__": 128035, |
| "__hi__": 128036, |
| "__hr__": 128037, |
| "__ht__": 128038, |
| "__hu__": 128039, |
| "__hy__": 128040, |
| "__id__": 128041, |
| "__ig__": 128042, |
| "__ilo__": 128043, |
| "__is__": 128044, |
| "__it__": 128045, |
| "__ja__": 128046, |
| "__jv__": 128047, |
| "__ka__": 128048, |
| "__kk__": 128049, |
| "__km__": 128050, |
| "__kn__": 128051, |
| "__ko__": 128052, |
| "__lb__": 128053, |
| "__lg__": 128054, |
| "__ln__": 128055, |
| "__lo__": 128056, |
| "__lt__": 128057, |
| "__lv__": 128058, |
| "__mg__": 128059, |
| "__mk__": 128060, |
| "__ml__": 128061, |
| "__mn__": 128062, |
| "__mr__": 128063, |
| "__ms__": 128064, |
| "__my__": 128065, |
| "__ne__": 128066, |
| "__nl__": 128067, |
| "__no__": 128068, |
| "__ns__": 128069, |
| "__oc__": 128070, |
| "__or__": 128071, |
| "__pa__": 128072, |
| "__pl__": 128073, |
| "__ps__": 128074, |
| "__pt__": 128075, |
| "__ro__": 128076, |
| "__ru__": 128077, |
| "__sd__": 128078, |
| "__si__": 128079, |
| "__sk__": 128080, |
| "__sl__": 128081, |
| "__so__": 128082, |
| "__sq__": 128083, |
| "__sr__": 128084, |
| "__ss__": 128085, |
| "__su__": 128086, |
| "__sv__": 128087, |
| "__sw__": 128088, |
| "__ta__": 128089, |
| "__th__": 128090, |
| "__tl__": 128091, |
| "__tn__": 128092, |
| "__tr__": 128093, |
| "__uk__": 128094, |
| "__ur__": 128095, |
| "__uz__": 128096, |
| "__vi__": 128097, |
| "__wo__": 128098, |
| "__xh__": 128099, |
| "__yi__": 128100, |
| "__yo__": 128101, |
| "__zh__": 128102, |
| "__zu__": 128103 |
| } |
|
|