diff --git "a/models/tokenizer/st_tokenizer_8k.vocab" "b/models/tokenizer/st_tokenizer_8k.vocab" new file mode 100644--- /dev/null +++ "b/models/tokenizer/st_tokenizer_8k.vocab" @@ -0,0 +1,8000 @@ + 0 + 0 + 0 + 0 +le -0 +ts -1 +▁b -2 +▁m -3 +▁s -4 +ng -5 +▁k -6 +la -7 +▁h -8 +▁le -9 +▁e -10 +▁a -11 +lo -12 +wa -13 +ha -14 +di -15 +▁ka -16 +na -17 +▁se -18 +▁ba -19 +▁ts -20 +ya -21 +▁ho -22 +▁bo -23 +▁mo -24 +▁o -25 +ho -26 +▁t -27 +▁p -28 +ne -29 +▁ya -30 +he -31 +li -32 +ri -33 +ka -34 +re -35 +▁sa -36 +▁ma -37 +▁di -38 +pa -39 +▁ha -40 +▁i -41 +te -42 +ra -43 +be -44 +ba -45 +▁f -46 +ro -47 +▁ke -48 +me -49 +mo -50 +hi -51 +▁wa -52 +ta -53 +se -54 +▁tse -55 +▁la -56 +▁tsa -57 +ma -58 +to -59 +ke -60 +▁c -61 +sa -62 +tse -63 +▁li -64 +we -65 +▁na -66 +ti -67 +ko -68 +▁af -69 +▁ile -70 +si -71 +▁n -72 +▁ea -73 +rika -74 +▁me -75 +tso -76 +ni -77 +so -78 +hlo -79 +tho -80 +ve -81 +no -82 +▁afrika -83 +▁ne -84 +tsa -85 +▁( -86 +rwa -87 +pe -88 +bo -89 +nd -90 +▁borwa -91 +▁j -92 +tsi -93 +po -94 +▁d -95 +hu -96 +mi -97 +ele -98 +ga -99 +▁hla -100 +▁hae -101 +▁re -102 +bi -103 +▁" -104 +▁the -105 +▁oa -106 +on -107 +pha -108 +leng -109 +go -110 +▁mme -111 +▁fe -112 +tj -113 +holo -114 +ge -115 +an -116 +▁u -117 +keng -118 +pala -119 +▁q -120 +fa -121 +tla -122 +hla -123 +lemo -124 +▁pele -125 +ki -126 +▁seba -127 +▁sehlo -128 +nt -129 +er -130 +▁pa -131 +pi -132 +or -133 +tjha -134 +▁so -135 +ny -136 +ru -137 +lu -138 +▁. -139 +lle -140 +lla -141 +tha -142 +tswa -143 +hwa -144 +bu -145 +ngo -146 +lwa -147 +▁to -148 +padi -149 +en -150 +st -151 +vi -152 +phe -153 +hle -154 +sebe -155 +". -156 +tlo -157 +nga -158 +▁bi -159 +oa -160 +uma -161 +ona -162 +ric -163 +▁kapa -164 +▁sehlopha -165 +sebetsi -166 +ved -167 +▁ko -168 +llo -169 +rie -170 +▁po -171 +▁hore -172 +▁motse -173 +ce -174 +fi -175 +▁pha -176 +nya -177 +pu -178 +tle -179 +ter -180 +▁w -181 +▁co -182 +th -183 +▁naha -184 +▁hlahi -185 +▁fi -186 +▁g -187 +neng -188 +the -189 +▁fa -190 +▁tlo -191 +▁sebe -192 +▁ca -193 +eng -194 +tseng -195 +tswe -196 +▁pro -197 +fo -198 +▁' -199 +▁of -200 +▁lo -201 +▁mase -202 +▁ret -203 +▁bakeng -204 +rieved -205 +▁retrieved -206 +ber -207 +▁y -208 +co -209 +▁- -210 +▁bapala -211 +▁mosebetsi -212 +fu -213 +ry -214 +▁selemo -215 +rolo -216 +▁qa -217 +▁de -218 +▁batho -219 +▁uni -220 +de -221 +hlodi -222 +in -223 +tu -224 +sotho -225 +thi -226 +▁mora -227 +▁phe -228 +▁in -229 +▁bu -230 +▁be -231 +▁nako -232 +lha -233 +▁fuma -234 +fe -235 +is -236 +▁sebapadi -237 +nye -238 +▁ra -239 +gu -240 +tshe -241 +▁pu -242 +▁moo -243 +lang -244 +cha -245 +▁hlo -246 +▁neng -247 +▁mehlodi -248 +al -249 +kga -250 +▁bolo -251 +▁afric -252 +za -253 +ngata -254 +lwe -255 +ku -256 +nse -257 +▁te -258 +▁tloha -259 +▁karolo -260 +▁masepala -261 +▁si -262 +▁boe -263 +▁kga -264 +▁su -265 +▁thu -266 +zi -267 +ll -268 +▁tla -269 +▁lefa -270 +roa -271 +tš -272 +▁bona -273 +▁lebo -274 +mora -275 +mu -276 +▁eo -277 +▁nang -278 +bela -279 +kgo -280 +thu -281 +▁sta -282 +▁etsa -283 +▁ro -284 +▁jo -285 +▁mao -286 +kolo -287 +du -288 +ty -289 +heng -290 +tion -291 +tjhaba -292 +▁maoto -293 +▁ileng -294 +▁v -295 +kho -296 +swa -297 +▁mu -298 +▁tha -299 +▁boroa -300 +▁pi -301 +▁du -302 +ch -303 +lao -304 +ngwa -305 +▁baa -306 +kha -307 +▁no -308 +she -309 +ci -310 +mino -311 +▁feti -312 +▁ta -313 +▁boi -314 +▁fo -315 +▁pe -316 +phi -317 +ca -318 +rs -319 +▁jwa -320 +▁dilemo -321 +▁haholo -322 +▁baahi -323 +sho -324 +kwa -325 +meng -326 +▁dipa -327 +ted -328 +▁al -329 +ja -330 +nda -331 +phelo -332 +▁qe -333 +ld -334 +ropo -335 +um -336 +▁kgo -337 +ly -338 +▁ja -339 +sisa -340 +▁sebaka -341 +ela -342 +▁sou -343 +wn -344 +▁tswa -345 +▁pre -346 +▁bophelo -347 +▁ga -348 +▁hlahileng -349 +gi -350 +▁phu -351 +jwa -352 +▁ju -353 +bedi -354 +da -355 +ea -356 +ss -357 +mber -358 +▁ar -359 +▁prove -360 +ck -361 +▁tshwa -362 +▁lesotho -363 +ria -364 +baka -365 +▁provense -366 +nong -367 +ndo -368 +hala -369 +▁fetisisa -370 +▁kge -371 +▁tshe -372 +▁african -373 +rd -374 +onal -375 +tsweng -376 +▁kha -377 +bitso -378 +▁kopa -379 +nts -380 +▁en -381 +▁mi -382 +bele -383 +han -384 +▁moe -385 +▁mmino -386 +rg -387 +▁vi -388 +▁wi -389 +▁hape -390 +▁united -391 +▁go -392 +▁and -393 +tshwa -394 +▁naheng -395 +tes -396 +tional -397 +▁botjha -398 +zu -399 +▁mong -400 +▁sete -401 +mela -402 +▁feta -403 +▁molao -404 +▁leng -405 +▁em -406 +▁bitswa -407 +burg -408 +teng -409 +▁ena -410 +tsatsi -411 +nes -412 +▁phaha -413 +), -414 +bini -415 +su -416 +▁loko -417 +disa -418 +▁qala -419 +nang -420 +polo -421 +pho -422 +▁thuto -423 +▁toropo -424 +reke -425 +reng -426 +▁on -427 +▁fumana -428 +▁botjhabela -429 +▁fapa -430 +▁south -431 +lane -432 +moho -433 +▁tl -434 +pele -435 +▁hlahe -436 +ua -437 +▁chi -438 +▁leboya -439 +▁ti -440 +▁bang -441 +mpo -442 +▁ama -443 +▁lea -444 +▁lebitso -445 +▁kamora -446 +). -447 +chi -448 +zulu -449 +���mma -450 +▁fre -451 +▁jwalo -452 +ntle -453 +▁we -454 +▁moka -455 +cto -456 +▁ki -457 +mong -458 +▁cha -459 +▁tseba -460 +▁tš -461 +tsoa -462 +ena -463 +▁afo -464 +▁nakong -465 +▁aforika -466 +ngwe -467 +▁tho -468 +ur -469 +ran -470 +pheng -471 +▁lena -472 +rea -473 +futa -474 +muso -475 +▁sekolo -476 +▁da -477 +▁unive -478 +▁sebetsa -479 +bamo -480 +▁puo -481 +tela -482 +▁tswe -483 +va -484 +dia -485 +kg -486 +mane -487 +px -488 +lli -489 +kelo -490 +▁tsejwa -491 +do -492 +tsoe -493 +▁kho -494 +▁new -495 +twa -496 +▁ge -497 +wi -498 +land -499 +▁sebini -500 +▁leha -501 +▁palo -502 +▁national -503 +hang -504 +▁mar -505 +▁fihle -506 +▁matla -507 +▁motho -508 +▁setso -509 +▁africa -510 +up -511 +kge -512 +▁ci -513 +chaba -514 +▁empa -515 +es -516 +tiki -517 +▁ngo -518 +▁hlahile -519 +one -520 +melo -521 +▁ame -522 +nyele -523 +sithi -524 +hileng -525 +moholo -526 +soa -527 +kare -528 +▁kenyele -529 +▁phahameng -530 +rt -531 +▁bapa -532 +nna -533 +shi -534 +lisa -535 +tsoeng -536 +▁letsatsi -537 +pali -538 +tata -539 +▁mae -540 +▁sena -541 +gue -542 +nal -543 +▁cape -544 +▁tlho -545 +lletsa -546 +▁ru -547 +▁tso -548 +▁lefatshe -549 +oho -550 +▁nka -551 +mpopo -552 +▁bana -553 +▁bitswang -554 +▁sehlopheng -555 +ngwana -556 +▁mokga -557 +▁for -558 +▁feto -559 +▁lefu -560 +▁moho -561 +sadi -562 +▁musi -563 +▁limpopo -564 +beng -565 +▁pla -566 +shome -567 +▁matjhaba -568 +▁he -569 +▁seo -570 +morao -571 +▁ngata -572 +fen -573 +hannes -574 +hareng -575 +▁fumane -576 +▁, -577 +man -578 +tlha -579 +▁tlha -580 +jo -581 +▁hi -582 +▁lu -583 +mier -584 +▁seng -585 +tic -586 +▁hloka -587 +mai -588 +ntlha -589 +polotiki -590 +▁premier -591 +▁fapaneng -592 +▁johannes -593 +▁au -594 +lela -595 +yo -596 +nte -597 +▁ntse -598 +▁papadi -599 +che -600 +limi -601 +raro -602 +phiri -603 +▁kang -604 +▁setjhaba -605 +▁setereke -606 +▁fetang -607 +▁league -608 +sta -609 +onte -610 +hleng -611 +▁lebaka -612 +ane -613 +tlala -614 +▁bobedi -615 +▁sebakeng -616 +ye -617 +son -618 +▁phato -619 +rom -620 +mmoho -621 +▁more -622 +▁nya -623 +▁bapalla -624 +long -625 +▁leto -626 +je -627 +▁fu -628 +hleke -629 +llwa -630 +▁johannesburg -631 +af -632 +▁z -633 +mba -634 +▁kwa -635 +▁yona -636 +ji -637 +taba -638 +▁hamorao -639 +mbi -640 +▁lefatš -641 +▁fihlela -642 +▁ku -643 +▁ri -644 +bane -645 +▁leka -646 +▁teng -647 +ngolo -648 +▁phere -649 +mpi -650 +nge -651 +hela -652 +sion -653 +▁dikga -654 +▁phupu -655 +▁phatlala -656 +▁an -657 +▁ding -658 +▁from -659 +▁bohlo -660 +▁dipapadi -661 +▁do -662 +▁je -663 +▁boetse -664 +den -665 +ster -666 +▁univesithi -667 +ste -668 +tli -669 +▁ve -670 +beli -671 +bile -672 +sh -673 +nde -674 +nia -675 +ctober -676 +▁kantle -677 +▁bapalang -678 +us -679 +puo -680 +▁mehleng -681 +bala -682 +rele -683 +lwetse -684 +kete -685 +▁con -686 +lweng -687 +▁r -688 +▁joa -689 +▁dina -690 +▁july -691 +▁thehi -692 +qa -693 +sia -694 +hele -695 +▁ntle -696 +▁october -697 +▁tu -698 +kola -699 +▁khe -700 +hlano -701 +thwa -702 +ward -703 +nyane -704 +▁hammoho -705 +▁isa -706 +▁tum -707 +▁tshi -708 +▁free -709 +▁tsoa -710 +▁music -711 +▁hlahetse -712 +bli -713 +nce -714 +fense -715 +▁dika -716 +kubele -717 +tswana -718 +▁sehla -719 +▁dikgau -720 +ef -721 +ue -722 +loa -723 +neo -724 +▁at -725 +▁ele -726 +tsang -727 +▁pina -728 +▁mohla -729 +▁nngwe -730 +cc -731 +pre -732 +▁or -733 +sang -734 +shene -735 +thollo -736 +▁basotho -737 +▁bohareng -738 +▁hlakubele -739 +gh -740 +▁– -741 +kwe -742 +▁is -743 +▁cup -744 +ngodi -745 +▁sowe -746 +▁state -747 +▁mpha -748 +▁lilemo -749 +▁mohope -750 +ptj -751 +ton -752 +folo -753 +sing -754 +▁feela -755 +▁sebedi -756 +dy -757 +pri -758 +rabe -759 +▁pho -760 +▁hona -761 +▁moru -762 +zer -763 +▁kg -764 +▁st -765 +etso -766 +▁city -767 +ban -768 +nwa -769 +ver -770 +rates -771 +▁town -772 +halang -773 +▁kgolo -774 +▁setho -775 +▁hangata -776 +ot -777 +▁[ -778 +lia -779 +per -780 +ropa -781 +▁ona -782 +▁award -783 +▁hara -784 +▁late -785 +▁mphalane -786 +", -787 +ls -788 +ufi -789 +▁archi -790 +▁batjha -791 +▁amerika -792 +xi -793 +loe -794 +olo -795 +▁eu -796 +▁man -797 +▁hlaha -798 +▁kenye -799 +hlanong -800 +▁mokgatlo -801 +com -802 +dio -803 +kgwa -804 +▁may -805 +▁dihlo -806 +▁mmuso -807 +ou -808 +hwe -809 +▁va -810 +dile -811 +hadi -812 +▁haufi -813 +for -814 +mme -815 +▁stu -816 +▁bile -817 +▁ithu -818 +▁poro -819 +▁leshome -820 +lele -821 +pina -822 +▁his -823 +▁ori -824 +ptjane -825 +▁albamo -826 +▁hlakola -827 +▁archived -828 +twe -829 +▁qo -830 +lione -831 +▁qete -832 +afrika -833 +▁tsamai -834 +▁tummeng -835 +▁phuptjane -836 +isa -837 +sti -838 +▁mai -839 +ginal -840 +▁eona -841 +▁lipa -842 +uni -843 +▁nove -844 +hokelo -845 +▁kholo -846 +▁thele -847 +▁basadi -848 +▁mefuta -849 +▁bophiri -850 +▁original -851 +el -852 +ol -853 +▁+ -854 +▁tj -855 +ment -856 +▁thi -857 +▁wor -858 +▁divi -859 +▁kgau -860 +▁metso -861 +▁lengolo -862 +am -863 +khu -864 +koa -865 +medi -866 +▁amo -867 +▁gau -868 +▁sebo -869 +▁chief -870 +▁ebile -871 +▁metsi -872 +▁profi -873 +▁soweto -874 +▁& -875 +ers -876 +theo -877 +ontein -878 +▁bopha -879 +▁colle -880 +▁lehla -881 +▁tsona -882 +ns -883 +ws -884 +khe -885 +wns -886 +dimo -887 +ress -888 +tsho -889 +▁khu -890 +▁she -891 +tions -892 +▁pirates -893 +mes -894 +▁moha -895 +▁maemo -896 +▁november -897 +▁mesebetsi -898 +ze -899 +ral -900 +hiso -901 +shoe -902 +kholo -903 +▁kene -904 +cember -905 +▁jwale -906 +chabela -907 +▁chiefs -908 +▁gauteng -909 +▁sechaba -910 +fane -911 +▁hapa -912 +▁lina -913 +▁lekgo -914 +▁uropa -915 +karetso -916 +▁za -917 +kopa -918 +▁kgu -919 +tsebi -920 +▁kena -921 +▁ntwa -922 +istata -923 +▁mmesa -924 +▁motsa -925 +▁morena -926 +▁december -927 +gra -928 +ball -929 +yane -930 +▁ste -931 +betso -932 +▁tseo -933 +▁boemo -934 +▁march -935 +▁ngwana -936 +▁states -937 +▁: -938 +tal -939 +hile -940 +▁kgwe -941 +▁joalo -942 +fontein -943 +▁boholo -944 +▁leboea -945 +▁college -946 +▁sehlooho -947 +▁sekolong -948 + -7942 +[ -7943 +ō -7944 +| -7945 +& -7946 +% -7947 +é -7948 ++ -7949 +ا -7950 +° -7951 +’ -7952 +ü -7953 += -7954 +ó -7955 +ô -7956 +ر -7957 +ç -7958 +! -7959 +á -7960 +ی -7961 +ş -7962 +í -7963 +? -7964 +ö -7965 +_ -7966 +è -7967 +ê -7968 +ğ -7969 +ˈ -7970 +م -7971 +ن -7972 +ت -7973 +‘ -7974 +” -7975 +} -7976 +“ -7977 +{ -7978 +ł -7979 +و -7980 +— -7981 +# -7982 +د -7983 +ل -7984 +ə -7985 +ه -7986 +ã -7987 +* -7988 +ج -7989 +ñ -7990 +ي -7991 +ý -7992 +ب -7993 +س -7994 +ú -7995