diff --git "a/models/tokenizer/bm_tokenizer_8k.vocab" "b/models/tokenizer/bm_tokenizer_8k.vocab" new file mode 100644--- /dev/null +++ "b/models/tokenizer/bm_tokenizer_8k.vocab" @@ -0,0 +1,8000 @@ + 0 + 0 + 0 + 0 +▁k -0 +an -1 +▁s -2 +▁b -3 +▁d -4 +▁n -5 +▁y -6 +▁m -7 +in -8 +▁f -9 +▁ka -10 +la -11 +ar -12 +▁t -13 +▁ye -14 +am -15 +on -16 +en -17 +▁a -18 +li -19 +▁c -20 +ri -21 +▁j -22 +▁la -23 +ti -24 +ba -25 +le -26 +ɔn -27 +gu -28 +go -29 +ya -30 +ra -31 +▁bɛ -32 +ara -33 +ugu -34 +▁ni -35 +un -36 +▁na -37 +▁w -38 +ana -39 +is -40 +▁min -41 +at -42 +ɔr -43 +ɛn -44 +▁ma -45 +▁o -46 +ma -47 +len -48 +re -49 +ani -50 +ko -51 +ka -52 +▁kɛ -53 +▁si -54 +ɔrɔ -55 +si -56 +lo -57 +▁( -58 +▁sa -59 +▁dugu -60 +ro -61 +▁ko -62 +amana -63 +ɔg -64 +▁san -65 +ga -66 +na -67 +rie -68 +il -69 +tion -70 +ama -71 +▁cat -72 +▁ani -73 +égo -74 +égorie -75 +▁catégorie -76 +ou -77 +▁p -78 +di -79 +▁ba -80 +▁don -81 +ni -82 +▁ta -83 +ul -84 +▁jamana -85 +▁di -86 +gi -87 +ɔgɔ -88 +▁h -89 +▁kan -90 +er -91 +▁ɲ -92 +▁u -93 +▁wa -94 +ɛr -95 +▁e -96 +▁i -97 +so -98 +ta -99 +kan -100 +ran -101 +▁saba -102 +▁se -103 +ɔnɔ -104 +da -105 +ogo -106 +bi -107 +▁de -108 +ki -109 +ur -110 +sa -111 +▁fɛ -112 +▁sabab -113 +ari -114 +lan -115 +▁da -116 +oro -117 +▁ch -118 +▁be -119 +▁fɔ -120 +▁g -121 +▁fa -122 +▁kɔ -123 +▁kɔnɔ -124 +es -125 +▁sababou -126 +▁in -127 +▁dɔ -128 +▁ke -129 +▁tun -130 +▁fran -131 +kɛ -132 +iri -133 +les -134 +ɛrɛ -135 +▁bama -136 +te -137 +us -138 +olo -139 +or -140 +▁is -141 +ire -142 +ille -143 +aman -144 +he -145 +▁sɔrɔ -146 +aran -147 +▁ku -148 +▁ti -149 +fɛ -150 +▁é -151 +niya -152 +ɛb -153 +▁bamako -154 +bara -155 +tions -156 +ais -157 +▁franç -158 +bn -159 +ini -160 +bam -161 +▁français -162 +se -163 +▁dic -164 +▁kelen -165 +▁char -166 +▁isbn -167 +▁ha -168 +um -169 +ditions -170 +▁donniya -171 +▁charles -172 +naire -173 +▁diction -174 +baille -175 +bambara -176 +bailleul -177 +▁éditions -178 +▁dictionnaire -179 +ke -180 +den -181 +eri -182 +nu -183 +aransi -184 +▁taa -185 +▁sigi -186 +▁bɔ -187 +▁an -188 +▁man -189 +▁bi -190 +inɛ -191 +▁ji -192 +fa -193 +lɛ -194 +ɲɛ -195 +▁fan -196 +gɛ -197 +▁af -198 +▁minnu -199 +▁mali -200 +ch -201 +mɛ -202 +). -203 +to -204 +de -205 +▁v -206 +▁kɔfɛ -207 +▁sira -208 +ji -209 +wa -210 +▁ja -211 +uru -212 +▁ɲɛ -213 +ulu -214 +▁" -215 +ɔrɔn -216 +lu -217 +▁baara -218 +bɔ -219 +ameri -220 +be -221 +jɛ -222 +▁yo -223 +▁mɔgɔ -224 +▁fara -225 +▁kun -226 +▁dan -227 +▁sen -228 +ye -229 +oli -230 +▁caman -231 +dugu -232 +▁mo -233 +wu -234 +ina -235 +ine -236 +ge -237 +▁fi -238 +▁cɛ -239 +ku -240 +▁mara -241 +ca -242 +yɔrɔ -243 +lɔ -244 +▁tɛ -245 +▁. -246 +man -247 +▁kuma -248 +bo -249 +bu -250 +tu -251 +▁faransi -252 +ce -253 +nen -254 +lima -255 +▁yɔrɔ -256 +▁to -257 +me -258 +qu -259 +▁tu -260 +▁dɔn -261 +ne -262 +don -263 +tigi -264 +ɔgɔn -265 +▁jamanaw -266 +px -267 +▁ɛn -268 +▁tɔgɔ -269 +▁woro -270 +▁su -271 +▁yɛ -272 +pa -273 +lon -274 +ent -275 +▁faso -276 +▁gu -277 +gan -278 +▁ameri -279 +▁ca -280 +▁do -281 +▁mu -282 +▁ya -283 +▁kɛra -284 +▁bɛɛ -285 +ɛnɛ -286 +yalen -287 +), -288 +▁worodugu -289 +ci -290 +cogo -291 +we -292 +▁mi -293 +▁nke -294 +▁ọ -295 +hi -296 +kolo -297 +▁fo -298 +▁afri -299 +▁ga -300 +ha -301 +ɛbɛn -302 +▁jon -303 +jamana -304 +▁walima -305 +do -306 +mu -307 +▁ban -308 +fan -309 +▁tle -310 +nan -311 +▁nin -312 +▁kɔrɔ -313 +▁kɛlɛ -314 +▁yɛrɛ -315 +▁so -316 +ban -317 +▁kalan -318 +heli -319 +▁cogo -320 +▁l -321 +tɛr -322 +uni -323 +▁kos -324 +inɛti -325 +▁fanga -326 +▁saheli -327 +tɛrinɛti -328 +ja -329 +wi -330 +▁dugum -331 +▁ɛntɛrinɛti -332 +▁dɛ -333 +▁re -334 +kili -335 +▁dugumogo -336 +▁yooro -337 +▁bo -338 +▁jɛ -339 +yiri -340 +▁fɔlɔ -341 +kun -342 +▁le -343 +ati -344 +▁r -345 +oni -346 +amara -347 +th -348 +yan -349 +▁ɲɔgɔn -350 +ate -351 +▁fana -352 +▁siti -353 +faransi -354 +du -355 +hụ -356 +st -357 +▁bolo -358 +al -359 +rô -360 +▁kon -361 +bin -362 +ɛbɛ -363 +▁z -364 +▁we -365 +▁kelenyalen -366 +ôrô -367 +▁masa -368 +ilan -369 +▁waati -370 +kọ -371 +▁du -372 +▁bon -373 +▁den -374 +▁fasojamana -375 +kɛla -376 +▁mans -377 +gh -378 +▁ki -379 +▁dam -380 +mi -381 +yi -382 +sen -383 +fi -384 +isi -385 +tin -386 +risi -387 +co -388 +▁sugu -389 +▁wala -390 +pu -391 +aba -392 +▁co -393 +▁tile -394 +▁jira -395 +▁tlebi -396 +▁mansamara -397 +kɔ -398 +vi -399 +gwu -400 +ɛgɛ -401 +▁wɛrɛ -402 +fin -403 +mɔgɔ -404 +▁me -405 +▁te -406 +denw -407 +▁afrika -408 +pe -409 +ata -410 +▁al -411 +▁ne -412 +▁amerika -413 +po -414 +dɔn -415 +af -416 +▁kɛmɛ -417 +.[ -418 +gin -419 +kulu -420 +▁kosɛbɛ -421 +ve -422 +ter -423 +▁jɔ -424 +▁jo -425 +▁nka -426 +amanan -427 +▁gun -428 +▁li -429 +▁vi -430 +▁tan -431 +▁jate -432 +que -433 +▁faaba -434 +▁sɛbɛn -435 +▁egwu -436 +▁: -437 +tan -438 +▁ci -439 +▁- -440 +▁di��ɛ -441 +fɔ -442 +pi -443 +▁yen -444 +▁walasa -445 +▁bamanan -446 +zi -447 +▁ar -448 +▁of -449 +▁bɔra -450 +yanfan -451 +kụ -452 +▁kè -453 +▁dɔw -454 +fe -455 +amu -456 +kɔnɔ -457 +▁bila -458 +▁bè -459 +ɛna -460 +urun -461 +▁olu -462 +▁kɔrɔn -463 +▁dugukolo -464 +ing -465 +et -466 +▁ro -467 +▁fɛn -468 +▁kɔn -469 +▁hakɛ -470 +mo -471 +▁no -472 +▁po -473 +dị -474 +gô -475 +kalo -476 +▁son -477 +▁kulu -478 +▁yiri -479 +bụ -480 +cɛ -481 +ft -482 +▁ju -483 +ɛrɛn -484 +ariya -485 +▁yoga -486 +as -487 +▁mar -488 +▁kô -489 +▁ali -490 +▁— -491 +tiki -492 +▁tɔn -493 +▁jago -494 +san -495 +iriki -496 +▁uk -497 +▁daminɛ -498 +▁dô -499 +▁afiriki -500 +▁sin -501 +lle -502 +bli -503 +dala -504 +▁bana -505 +▁sababu -506 +▁“ -507 +kɔn -508 +ɛri -509 +▁fô -510 +wift -511 +▁fil -512 +▁mis -513 +ry -514 +▁ŋ -515 +▁aw -516 +uman -517 +▁swift -518 +▁mari -519 +▁bé -520 +tigɛ -521 +▁kɔgɔ -522 +sh -523 +▁jan -524 +▁fila -525 +▁jɛkulu -526 +ôgô -527 +ariku -528 +▁kura -529 +▁tuma -530 +▁wele -531 +▁con -532 +▁pro -533 +▁poli -534 +▁suguya -535 +▁fe -536 +▁tɔ -537 +▁kɛr -538 +▁boli -539 +su -540 +dan -541 +▁dị -542 +kụkọ -543 +▁the -544 +▁bɛna -545 +▁wɛrɛw -546 +▁kɔgɔji -547 +yo -548 +kaw -549 +wan -550 +▁sis -551 +▁kɛnɛ -552 +▁ukra -553 +au -554 +min -555 +fara -556 +jago -557 +▁bin -558 +▁sôrô -559 +▁ɲɛmɔgɔ -560 +▁ɔ -561 +ene -562 +kɔrɔ -563 +▁ce -564 +▁nɔ -565 +▁on -566 +▁muso -567 +▁sɛnɛ -568 +▁dilan -569 +ll -570 +tt -571 +ɲi -572 +ịhụ -573 +▁fari -574 +▁bamanankan -575 +ali -576 +gne -577 +nanya -578 +ịhụnanya -579 +▁bu -580 +▁lo -581 +▁des -582 +▁kab -583 +▁с -584 +▁lam -585 +▁sariya -586 +pụ -587 +▁et -588 +baga -589 +▁fin -590 +▁jɛn -591 +je -592 +▁ụ -593 +our -594 +ron -595 +taa -596 +▁franse -597 +εε -598 +▁[ -599 +▁« -600 +ami -601 +ega -602 +fɛn -603 +▁ho -604 +bolo -605 +▁laban -606 +▁hakili -607 +eni -608 +▁ahụ -609 +▁ndị -610 +amerika -611 +ru -612 +ty -613 +té -614 +dio -615 +sin -616 +▁pa -617 +mali -618 +ropa -619 +▁lab -620 +▁foro -621 +▁yiriwa -622 +▁farafin -623 +law -624 +ver -625 +▁naf -626 +ts -627 +ard -628 +dia -629 +▁fu -630 +▁/ -631 +▁at -632 +▁daɲɛ -633 +.. -634 +ww -635 +ere -636 +nin -637 +▁en -638 +▁go -639 +▁yé -640 +▁tra -641 +▁kama -642 +ɔrɔnya -643 +▁senega -644 +▁– -645 +int -646 +son -647 +▁bụ -648 +▁sh -649 +▁wi -650 +ulon -651 +kɛlaw -652 +kɛrɛn -653 +▁siri -654 +▁tariku -655 +▁baarakɛ -656 +▁+ -657 +sia -658 +▁baw -659 +▁dabɔ -660 +▁marifa -661 +rụ -662 +на -663 +▁, -664 +las -665 +bali -666 +jɛgɛ -667 +yali -668 +▁taama -669 +▁kɛmɛda -670 +▁sigilen -671 +rè -672 +che -673 +kow -674 +dama -675 +lenw -676 +▁kolon -677 +▁mɔgɔw -678 +▁sigira -679 +▁ameriki -680 +▁jamanadenw -681 +▁ɛ -682 +pan -683 +▁lu -684 +▁un -685 +▁duni -686 +▁lajɛ -687 +▁laka -688 +▁sila -689 +èn -690 +ск -691 +ena -692 +▁aka -693 +▁gine -694 +ра -695 +ber -696 +▁es -697 +▁wo -698 +▁dun -699 +▁laa -700 +publi -701 +▁cent -702 +▁tugu -703 +dɔnnen -704 +▁kɛrɛn -705 +▁bamakɔ -706 +▁jɔyɔrɔ -707 +▁politiki -708 +▁в -709 +are -710 +baw -711 +naf -712 +▁dé -713 +▁nt -714 +▁ra -715 +land -716 +tara -717 +▁tali -718 +▁wale -719 +▁cogoya -720 +▁masala -721 +kɛrɛnnen -722 +ré -723 +то -724 +mis -725 +▁je -726 +afri -727 +▁mas -728 +▁mili -729 +▁sisan -730 +el -731 +fo -732 +ps -733 +chi -734 +kar -735 +yɛn -736 +▁kaba -737 +▁minɛ -738 +▁site -739 +▁к -740 +jan -741 +res -742 +ton -743 +▁bar -744 +▁bɛn -745 +▁nafa -746 +▁kɛrɛnkɛrɛnnen -747 +ph -748 +ɛni -749 +hara -750 +▁ihe -751 +arabu -752 +bagan -753 +kalan -754 +▁kɔkan -755 +▁mansa -756 +▁kabini -757 +ck -758 +baa -759 +bia -760 +mba -761 +tra -762 +▁sun -763 +▁fanba -764 +▁tulon -765 +▁kunnaf -766 +▁farafinna -767 +lé -768 +va -769 +ces -770 +cha -771 +lin -772 +mar -773 +tte -774 +▁ale -775 +▁fla -776 +sɛbɛn -777 +▁sika -778 +ال -779 +tik -780 +ment -781 +tali -782 +▁nan -783 +▁hali -784 +▁tila -785 +afrika -786 +▁eropa -787 +▁laadala -788 +▁ịhụnanya -789 +▁jamanaden -790 +kw -791 +▁hi -792 +▁sɛ -793 +bila -794 +▁abụ -795 +▁mun -796 +▁sou -797 +▁cɛma -798 +▁kalo -799 +▁ukraine -800 +ct -801 +tɛ -802 +ɲa -803 +▁л -804 +▁р -805 +babu -806 +sira -807 +▁kar -808 +▁nana -809 +▁yɛlɛ -810 +▁danbɔ -811 +▁irisi -812 +▁sahara -813 +▁sigida -814 +▁kunnafoni -815 +za -816 +rit -817 +èrè -818 +▁sɛb -819 +ugula -820 +ɛnajɛ -821 +▁wari -822 +▁bonya -823 +▁filmu -824 +tɔ -825 +▁; -826 +▁th -827 +▁sam -828 +▁uni -829 +kelen -830 +kɔrɔn -831 +▁fɔli -832 +▁fɛɛrɛ -833 +▁sɛben -834 +▁hadama -835 +▁danfara -836 +по -837 +▁| -838 +liw -839 +ste -840 +tes -841 +uli -842 +ence -843 +lanw -844 +��lib -845 +▁kilo -846 +▁kolo -847 +▁tiɲɛ -848 +// -849 +vo -850 +▁ʃ -851 +can -852 +mes -853 +ɛlɛ -854 +▁as -855 +▁pe -856 +dasi -857 +fasi -858 +wari -859 +▁ala -860 +▁mon -861 +▁yan -862 +▁ɲin -863 +▁dɔgɔ -864 +▁jugu -865 +▁minu -866 +▁musa -867 +▁jigin -868 +▁saint -869 +kuntigi -870 +:// -871 +ilɛ -872 +tɔn -873 +bola -874 +gera -875 +▁dala -876 +▁tɛmɛ -877 +▁akụkọ -878 +▁faran -879 +▁senkan -880 +▁farikolo -881 +mb -882 +sɛ -883 +des -884 +han -885 +mah -886 +nes -887 +▁nk -888 +▁pi -889 +ania -890 +▁art -891 +▁bam -892 +▁kul -893 +▁par -894 +▁dɛmɛ -895 +▁sini -896 +▁dunia -897 +▁sɔrɔdasi -898 +no -899 +arị -900 +der -901 +▁ex -902 +▁nu -903 +bili -904 +▁hak -905 +donna -906 +mancɛ -907 +tinin -908 +ɛnama -909 +▁kɛɲɛ -910 +▁seko -911 +▁sera -912 +cation -913 +▁atlan -914 +▁kongo -915 +▁foroba -916 +▁fitinin -917 +dɔ -918 +nô -919 +tle -920 +usu -921 +▁am -922 +▁au -923 +▁er -924 +▁gi -925 +buru -926 +sɛgɛ -927 +▁ama -928 +▁jɔn -929 +▁wee -930 +▁fili -931 +▁nɛgɛ -932 +▁sara -933 +▁sɛgɛ -934 +▁bagan -935 +▁kungo -936 +▁segin -937 +▁africa -938 +▁nafolo -939 +▁jamanakuntigi -940 +ен -941 +▁* -942 +▁» -943 +lar -944 +yɔn -945 +ɔra -946 +▁fè -947 +paɲi -948 +▁mus -949 +▁tum -950 +▁bara -951 +▁ɛndu -952 +▁ebola -953 +▁galle -954 +▁kɛnɛya -955 +▁sɔrɔko -956 +▁yɛlɛma -957 +▁cɛmancɛ -958 +▁dɔnkili -959 +▁yɛrɛmah -960 +▁madɔnnen -961 +▁sigiyɔrɔ -962 +ho -963 +aka -964 +ben -965 +ger -966 +gon -967 +mɛn -968 +orr -969 +run -970 +sun -971 +tto -972 +urs -973 +▁bô -974 +▁là -975 +▁sé -976 +▁ال -977 +pụta -978 +were -979 +▁and -980 +▁dum -981 +▁for -982 +▁les -983 +▁por -984 +akụkọ -985 +▁jama -986 +▁kara -987 +▁lase -988 +▁mugu -989 +▁sanu -990 +gnette -991 +▁diinɛ -992 +▁kɛcogo -993 +▁masakɛ -994 +". -995 +fu -996 +gɔ -997 +it -998 +ов -999 +dew -1000 +kis -1001 +sis -1002 +sti -1003 +umb -1004 +ɔni -1005 +▁ge -1006 +▁tè -1007 +sion -1008 +tɛrɛ -1009 +▁kala -1010 +▁wati -1011 +orrent -1012 +▁kataa -1013 +▁sudan -1014 +▁woroduguyanfan -1015 +ak -1016 +uk -1017 +▁у -1018 +▁‘ -1019 +ala -1020 +ang -1021 +bur -1022 +ral -1023 +the -1024 +▁ak -1025 +▁el -1026 +ɔnin -1027 +baara -1028 +▁kono -1029 +▁môgô -1030 +tional -1031 +▁arabu -1032 +▁inter -1033 +▁labɛn -1034 +▁surun -1035 +▁ɲuman -1036 +▁kalanso -1037 +▁sirilanw -1038 +▁sɛgɛsɛgɛ -1039 +né -1040 +pp -1041 +ss -1042 +ية -1043 +▁< -1044 +sɛn -1045 +▁qu -1046 +faso -1047 +▁com -1048 +▁shi -1049 +bilen -1050 +ɲɔgɔn -1051 +▁jiri -1052 +▁kore -1053 +▁onye -1054 +▁risi -1055 +▁togo -1056 +▁yoro -1057 +▁dɔnko -1058 +▁hakil -1059 +▁motto -1060 +▁ninnu -1061 +▁silamɛ -1062 +▁faranfasi -1063 +bɛ -1064 +jo -1065 +ld -1066 +fil -1067 +gar -1068 +kon -1069 +ria -1070 +ɔpu -1071 +fina -1072 +jata -1073 +main -1074 +tini -1075 +ɲini -1076 +gharị -1077 +▁dɔrɔn -1078 +▁kulɛri -1079 +▁lakali -1080 +▁ɲinini -1081 +▁dɔnniya -1082 +▁gallery -1083 +▁kumajago -1084 + -7835 +я -7836 +ل -7837 ++ -7838 +ь -7839 +% -7840 +ʃ -7841 +б -7842 +ي -7843 +ā -7844 +ч -7845 +ò -7846 +ة -7847 +* -7848 += -7849 +ц -7850 +ر -7851 +ᴐ -7852 +‘ -7853 +م -7854 +! -7855 +· -7856 +í -7857 +ï -7858 +ї -7859 +â -7860 +х -7861 +ú -7862 +ù -7863 +ы -7864 +ф -7865 +ш -7866 +_ -7867 +ж -7868 +و -7869 +? -7870 +ߊ -7871 +ƙ -7872 +ю -7873 +̀ -7874 +á -7875 +ɳ -7876 +ó -7877 +ب -7878 +ɗ -7879 +& -7880 +ü -7881 +є -7882 +ì -7883 +ê -7884 +ج -7885 +ë -7886 +ş -7887 +س -7888 +ع -7889 +ن -7890 +ߞ -7891 +ñ -7892 +ه -7893 +ߏ -7894 +ߒ -7895 +߬ -7896 +ī -7897 +ъ -7898 +э -7899 +ت -7900 +د -7901 +ߋ -7902 +ߘ -7903 +߫ -7904 +߲ -7905 +ı -7906 +œ -7907 +ś -7908 +ū -7909 +ߓ -7910 +° -7911 +İ -7912 +ߌ -7913 +ὲ -7914 +ⴰ -7915 +日 -7916 +î -7917 +ö -7918 +́ -7919 +إ -7920 +ش -7921 +ߍ -7922 +ߎ -7923 +‰ -7924 +{ -7925 +ä -7926 +ø -7927 +щ -7928 +י -7929 +् -7930 +ṇ -7931 +ⵜ -7932 +本 -7933 +# -7934 +ã -7935 +û -7936 +ć -7937 +غ -7938 +ك -7939 +ߐ -7940 +ߙ -7941 +ߝ -7942 +ߡ -7943 +社 -7944 +} -7945 +ł -7946 +ũ -7947 +λ -7948 +ѣ -7949 +ئ -7950 +ز -7951 +ߖ -7952 +ߛ -7953 +ߟ -7954 +ߠ -7955 +ߣ -7956 +ߦ -7957 +ध -7958 +म -7959 +ố -7960 +′ -7961 +『 -7962 +』 -7963 +の -7964 +史 -7965 +朝 -7966 +英 -7967 +集 -7968 +$ -7969 +^ -7970 +£ -7971 +× -7972 +å -7973 +ą -7974 +č -7975 +š -7976 +ʻ -7977 +ˈ -7978 +ː -7979 +δ -7980 +σ -7981 +ה -7982 +ו -7983 +ל -7984 +מ -7985 +ר -7986 +ת -7987 +، -7988 +ح -7989 +ص -7990 +ظ -7991 +ق -7992 +ى -7993 +َ -7994 +ग -7995