diff --git "a/models/tokenizer/haw_tokenizer_8k.vocab" "b/models/tokenizer/haw_tokenizer_8k.vocab" new file mode 100644--- /dev/null +++ "b/models/tokenizer/haw_tokenizer_8k.vocab" @@ -0,0 +1,8000 @@ + 0 + 0 + 0 + 0 +▁k -0 +▁m -1 +na -2 +▁h -3 +▁ka -4 +▁ma -5 +▁ʻ -6 +▁i -7 +ka -8 +▁a -9 +▁p -10 +le -11 +▁n -12 +la -13 +▁o -14 +▁me -15 +ia -16 +ua -17 +ana -18 +ik -19 +▁nā -20 +▁ke -21 +lo -22 +▁ho -23 +▁he -24 +ni -25 +ma -26 +li -27 +▁s -28 +wa -29 +▁mea -30 +ha -31 +hi -32 +▁e -33 +pa -34 +on -35 +▁pa -36 +▁ua -37 +▁ha -38 +ui -39 +lu -40 +▁c -41 +lā -42 +ke -43 +▁mai -44 +iki -45 +▁ko -46 +er -47 +ne -48 +pu -49 +▁ki -50 +oh -51 +▁kū -52 +ole -53 +▁b -54 +ti -55 +ko -56 +▁ia -57 +▁( -58 +ra -59 +▁kā -60 +ku -61 +▁lo -62 +naka -63 +▁maka -64 +▁w -65 +ho -66 +▁le -67 +no -68 +ike -69 +au -70 +me -71 +▁t -72 +▁d -73 +hiki -74 +▁no -75 +▁makahiki -76 +he -77 +▁wa -78 +kā -79 +▁nui -80 +nia -81 +▁mo -82 +▁‘ -83 +as -84 +ki -85 +puni -86 +▁la -87 +▁hana -88 +▁au -89 +ika -90 +ohana -91 +re -92 +▁ku -93 +▁helu -94 +mo -95 +ōle -96 +or -97 +▁se -98 +▁aupuni -99 +pe -100 +ōlelo -101 +▁f -102 +▁lā -103 +▁pi -104 +ri -105 +wai -106 +os -107 +▁kona -108 +lle -109 +▁haka -110 +ina -111 +▁hakakā -112 +▁pu -113 +▁loa -114 +▁kahi -115 +▁pana -116 +▁panalā -117 +pania -118 +▁mau -119 +▁cas -120 +al -121 +▁pā -122 +▁sepania -123 +ikā -124 +hu -125 +▁po -126 +▁kiw -127 +▁kiwikā -128 +lana -129 +▁g -130 +▁casti -131 +▁pū -132 +▁leon -133 +kala -134 +▁castille -135 +nā -136 +nt -137 +mu -138 +nd -139 +ur -140 +▁, -141 +▁l -142 +kahi -143 +ng -144 +▁kē -145 +▁hui -146 +hana -147 +▁hawai -148 +▁pe -149 +▁ana -150 +st -151 +▁u -152 +kua -153 +▁hō -154 +▁ne -155 +ro -156 +kik -157 +▁ai -158 +mi -159 +▁— -160 +ano -161 +ai -162 +we -163 +lika -164 +▁kanaka -165 +▁na -166 +kikī -167 +▁li -168 +oni -169 +▁kekahi -170 +ona -171 +akikī -172 +ca -173 +▁mana -174 +▁j -175 +mole -176 +oi -177 +▁kānaka -178 +▁kūmole -179 +el -180 +āina -181 +pi -182 +▁mua -183 +▁sa -184 +urg -185 +▁kēia -186 +kou -187 +āna -188 +ta -189 +za -190 +▁kau -191 +▁- -192 +▁burg -193 +ani -194 +ame -195 +kau -196 +▁wā -197 +uha -198 +▁burgos -199 +lama -200 +▁de -201 +▁maika -202 +▁kula -203 +▁v -204 +uhale -205 +te -206 +▁iā -207 +▁like -208 +nu -209 +ono -210 +kalani -211 +▁" -212 +▁mā -213 +hai -214 +▁hiki -215 +il -216 +▁pala -217 +amelika -218 +ar -219 +nakauhale -220 +oma -221 +pika -222 +▁inā -223 +▁pukalani -224 +kema -225 +lanakauhale -226 +▁moku -227 +▁kūlanakauhale -228 +nua -229 +▁mele -230 +es -231 +▁the -232 +en -233 +▁aku -234 +▁kāna -235 +iko -236 +▁pahu -237 +▁piha -238 +us -239 +om -240 +▁lu -241 +▁mi -242 +kele -243 +▁of -244 +pikala -245 +▁kapikala -246 +hopu -247 +▁za -248 +▁pahuhopu -249 +▁hele -250 +hine -251 +ama -252 +▁hope -253 +is -254 +oho -255 +pala -256 +oku -257 +olelo -258 +mora -259 +ena -260 +▁zamora -261 +▁hou -262 +kū -263 +▁komo -264 +ānau -265 +ce -266 +▁loko -267 +um -268 +ón -269 +de -270 +▁lākou -271 +▁papa -272 +lia -273 +da -274 +ula -275 +▁re -276 +oihana -277 +▁kah -278 +▁wai -279 +oli -280 +▁pele -281 +▁león -282 +ou -283 +▁kumu -284 +▁aia -285 +pana -286 +nca -287 +▁pō -288 +▁hale -289 +oria -290 +noa -291 +▁wo -292 +an -293 +ve -294 +tion -295 +oka -296 +lamanca -297 +▁salamanca -298 +▁pili -299 +▁akua -300 +▁r -301 +▁soria -302 +th -303 +ulo -304 +▁we -305 +▁manawa -306 +nui -307 +▁luna -308 +mā -309 +to -310 +▁ala -311 +ili -312 +▁hua -313 +po -314 +▁honua -315 +▁kūlana -316 +▁kaulana -317 +▁ola -318 +okumu -319 +▁hānau -320 +oko -321 +▁palapala -322 +nae -323 +ic -324 +ter -325 +eki -326 +▁y -327 +▁inoa -328 +▁kom -329 +ue -330 +▁makema -331 +rd -332 +▁kapa -333 +vi -334 +▁hā -335 +ss -336 +olo -337 +omaka -338 +wi -339 +ala -340 +▁komohana -341 +aka -342 +▁mō -343 +▁kī -344 +▁makemake -345 +▁su -346 +ola -347 +▁makua -348 +▁kākau -349 +di -350 +▁waho -351 +ch -352 +als -353 +ers -354 +▁noho -355 +). -356 +▁lilo -357 +ele -358 +▁ali -359 +co -360 +▁co -361 +▁kela -362 +alshai -363 +▁maoli -364 +mb -365 +per -366 +tu -367 +▁hi -368 +ns -369 +ts -370 +▁nei -371 +ge -372 +▁ro -373 +kila -374 +ila -375 +▁pau -376 +kanu -377 +ck -378 +▁ca -379 +▁meakanu -380 +▁ri -381 +sa -382 +ld -383 +va -384 +ty -385 +ga -386 +kia -387 +mai -388 +▁and -389 +▁ra -390 +lanahai -391 +px -392 +▁be -393 +▁wahi -394 +▁to -395 +ao -396 +▁' -397 +▁hema -398 +), -399 +▁nō -400 +▁kahua -401 +▁al -402 +ale -403 +▁kō -404 +āpana -405 +ci -406 +▁mar -407 +ba -408 +ulou -409 +▁nānā -410 +▁loulou -411 +ohe -412 +uli -413 +▁ni -414 +helu -415 +▁holo -416 +ian -417 +▁ar -418 +pono -419 +▁make -420 +▁waena -421 +se -422 +hane -423 +wele -424 +▁pūnae -425 +▁wahine -426 +▁puke -427 +una -428 +ākau -429 +▁iho -430 +▁pono -431 +in -432 +▁fa -433 +nter -434 +▁lakou -435 +▁pūnaewele -436 +▁kai -437 +▁uhane -438 +▁in -439 +ry -440 +▁wiki -441 +kike -442 +▁paha -443 +▁mokupuni -444 +do -445 +▁mālama -446 +▁ba -447 +▁akā -448 +ulu -449 +loha -450 +maika -451 +▁keaka -452 +▁keiki -453 +▁kaha -454 +▁. -455 +lua -456 +opa -457 +▁lawe -458 +▁« -459 +ikua -460 +kel -461 +hele -462 +▁da -463 +▁lāua -464 +▁kahiko -465 +▁() -466 +▁sta -467 +ran -468 +meha -469 +ololi -470 +▁super -471 +lona -472 +ul -473 +ber -474 +▁con -475 +▁kama -476 +▁hau -477 +▁alaka -478 +▁kaua -479 +▁kele -480 +kikena -481 +▁kepa -482 +ly -483 +kalama -484 +lai -485 +hina -486 +fa -487 +ene -488 +io -489 +ru -490 +ūa -491 +▁(ʻ -492 +▁ja -493 +▁pelekikena -494 +hā -495 +mana -496 +▁wale -497 +īa -498 +man -499 +lani -500 +▁cha -501 +▁hōkū -502 +eri -503 +nuali -504 +". -505 +▁z -506 +nte -507 +hili -508 +▁nekele -509 +bo -510 +▁wi -511 +▁awa -512 +▁pil -513 +olaha -514 +ok -515 +wī -516 +▁is -517 +▁makani -518 +ea -519 +tt -520 +▁bo -521 +▁thu -522 +▁kana -523 +▁koho -524 +▁kulanui -525 +oia -526 +omana -527 +elua -528 +▁aloha -529 +▁kūhelu -530 +mar -531 +tor -532 +▁ianuali -533 +go -534 +ela -535 +hon -536 +que -537 +cha -538 +chi -539 +iku -540 +▁do -541 +nō -542 +▁fi -543 +▁go -544 +▁polo -545 +br -546 +▁thumb -547 +▁peleka -548 +gh -549 +si -550 +ape -551 +▁kēlā -552 +▁lana -553 +ed -554 +puka -555 +▁ti -556 +ome -557 +kana -558 +▁kōkua -559 +▁vi -560 +illa -561 +▁waiho -562 +▁haku -563 +ya -564 +awi -565 +sia -566 +use -567 +▁dra -568 +▁iāpana -569 +▁kaikua -570 +▁lanakila -571 +mapa -572 +kemapa -573 +▁laila -574 +ll -575 +heu -576 +land -577 +mon -578 +ver -579 +moku -580 +hala -581 +īama -582 +▁hoa -583 +▁lalo -584 +aukala -585 +▁so -586 +▁wokele -587 +amau -588 +ako -589 +be -590 +rt -591 +▁oia -592 +▁award -593 +com -594 +kulu -595 +ment -596 +▁kālai -597 +▁wikiō -598 +ikia -599 +omeheu -600 +aha -601 +aua -602 +kina -603 +maka -604 +▁ā -605 +ndo -606 +▁car -607 +▁makana -608 +▁cu -609 +▁lua -610 +naehana -611 +kī -612 +▁+ -613 +uki -614 +▁ge -615 +▁fran -616 +▁haum -617 +▁hula -618 +hui -619 +ros -620 +▁si -621 +loko -622 +ōnaehana -623 +alka -624 +▁pale -625 +▁pāhili -626 +▁x -627 +epe -628 +▁iu -629 +auao -630 +▁kālā -631 +▁kīwī -632 +▁wokel -633 +bi -634 +pp -635 +hou -636 +shi -637 +opuka -638 +▁waiwai -639 +gi -640 +era -641 +pene -642 +waho -643 +▁kūkulu -644 + -2824 +▁q -2825 +▁ž -2826 +▁ἡ -2827 +apa -2828 +bit -2829 +dio -2830 +gua -2831 +iau -2832 +ile -2833 +lar -2834 +pho -2835 +pro -2836 +rac -2837 +sou -2838 +tus -2839 +ukē -2840 +vil -2841 +vin -2842 +viv -2843 +wal -2844 +zie -2845 +āhu -2846 +▁gi -2847 +▁io -2848 +▁zu -2849 +alth -2850 +berg -2851 +cebo -2852 +cula -2853 +eler -2854 +essi -2855 +gger -2856 +hiro -2857 +hope -2858 +iama -2859 +ikal -2860 +ject -2861 +kaha -2862 +lapa -2863 +lock -2864 +meka -2865 +mont -2866 +ndes -2867 +ness -2868 +nton -2869 +oshi -2870 +pera -2871 +pher -2872 +phic -2873 +pila -2874 +pita -2875 +rade -2876 +talo -2877 +tist -2878 +wapa -2879 +wber -2880 +▁dic -2881 +▁dog -2882 +▁est -2883 +▁fur -2884 +▁mos -2885 +▁non -2886 +▁rey -2887 +▁rob -2888 +▁sun -2889 +▁tag -2890 +▁tol -2891 +▁vol -2892 +amapa -2893 +briel -2894 +maiki -2895 +ohōkā -2896 +tures -2897 +▁alde -2898 +▁anda -2899 +▁anti -2900 +▁baro -2901 +▁bili -2902 +▁dale -2903 +▁dora -2904 +▁hanu -2905 +▁kids -2906 +▁lion -2907 +▁loio -2908 +▁love -2909 +▁manō -2910 +▁meia -2911 +▁mist -2912 +▁name -2913 +▁nīna -2914 +▁pahi -2915 +▁paul -2916 +▁pete -2917 +▁pāku -2918 +▁rica -2919 +▁rita -2920 +▁rosa -2921 +▁team -2922 +▁vega -2923 +▁wind -2924 +molele -2925 +nesian -2926 +olaule -2927 +rotiko -2928 +▁actor -2929 +▁awāwa -2930 +▁beast -2931 +▁canta -2932 +▁chris -2933 +▁disco -2934 +▁ferna -2935 +▁genus -2936 +▁georg -2937 +▁kāmoa -2938 +▁mamao -2939 +▁māoli -2940 +▁nūhou -2941 +▁pedro -2942 +▁roman -2943 +▁trans -2944 +kānāwai -2945 +▁antoni -2946 +▁hakaka -2947 +▁kepani -2948 +▁philip -2949 +▁pāpaho -2950 +▁pōpeku -2951 +▁totoro -2952 +▁biliona -2953 +▁hapalua -2954 +▁maluhia -2955 +▁website -2956 +erusalema -2957 +ohōkālani -2958 +okolokolo -2959 +▁aotearoa -2960 +▁hemaiala -2961 +▁hemolele -2962 +▁kaiapuni -2963 +▁mcdonald -2964 +▁miyamoto -2965 +▁strawber -2966 +▁alexander -2967 +▁christmas -2968 +▁hunakalia -2969 +▁institute -2970 +▁sakaguchi -2971 +▁technology -2972 +▁interlingua -2973 +▁kumukānāwai -2974 +hy -2975 +pm -2976 +td -2977 +ér -2978 +ís -2979 +▁~ -2980 +▁á -2981 +▁τ -2982 +▁м -2983 +cou -2984 +deo -2985 +dor -2986 +eli -2987 +ica -2988 +jos -2989 +ldo -2990 +lez -2991 +lon -2992 +mba -2993 +nio -2994 +pre -2995 +rda -2996 +rew -2997 +rez -2998 +rst -2999 +sen -3000 +ser -3001 +tto -3002 +tur -3003 +wan -3004 +zon -3005 +▁bc -3006 +▁br -3007 +▁nē -3008 +▁nī -3009 +▁pá -3010 +▁ty -3011 +algi -3012 +auty -3013 +ddle -3014 +esti -3015 +gaku -3016 +ging -3017 +gram -3018 +hima -3019 +hula -3020 +hune -3021 +kian -3022 +lelo -3023 +lish -3024 +mare -3025 +ntes -3026 +pani -3027 +papa -3028 +pena -3029 +pōpō -3030 +ries -3031 +saka -3032 +stin -3033 +tice -3034 +tins -3035 +tone -3036 +view -3037 +ícia -3038 +ória -3039 +āmai -3040 +▁ace -3041 +▁bin -3042 +▁bio -3043 +▁bpm -3044 +▁bre -3045 +▁dal -3046 +▁ene -3047 +▁eth -3048 +▁gau -3049 +▁geo -3050 +▁ili -3051 +▁jan -3052 +▁kar -3053 +▁ker -3054 +▁lee -3055 +▁lou -3056 +▁max -3057 +▁nas -3058 +▁nfc -3059 +▁peo -3060 +▁pol -3061 +▁pop -3062 +▁sal -3063 +▁sar -3064 +▁shō -3065 +▁sus -3066 +▁tur -3067 +▁wel -3068 +asala -3069 +aster -3070 +aurus -3071 +craft -3072 +ences -3073 +erato -3074 +ikuma -3075 +kolio -3076 +matsu -3077 +nesia -3078 +olilo -3079 +ports -3080 +tical -3081 +zioni -3082 +ōlapa -3083 +▁asia -3084 +▁camp -3085 +▁dead -3086 +▁esco -3087 +▁gula -3088 +▁huna -3089 +▁kalā -3090 +▁kuni -3091 +▁kūpa -3092 +▁marc -3093 +▁mika -3094 +▁neke -3095 +▁nole -3096 +▁poku -3097 +▁prof -3098 +▁pulu -3099 +▁saga -3100 +▁step -3101 +▁wolf -3102 +ikiana -3103 +iuwapa -3104 +kanasā -3105 +omākau -3106 +panese -3107 +tugala -3108 +▁akula -3109 +▁conan -3110 +▁crist -3111 +▁globe -3112 +▁globo -3113 +▁goiás -3114 +▁hanks -3115 +▁johan -3116 +▁keawe -3117 +▁lehua -3118 +▁monte -3119 +▁perfe -3120 +▁prize -3121 +▁rodri -3122 +▁sarah -3123 +▁tōkiō -3124 +▁ukali -3125 +▁ukana -3126 +▁ulana -3127 +▁žalgi -3128 +gentina -3129 +olution -3130 +opakele -3131 +tadores -3132 +umikuma -3133 +▁actors -3134 +▁austin -3135 +▁baluna -3136 +▁batman -3137 +▁beamer -3138 +▁comedy -3139 +▁hiamoe -3140 +▁hōkūle -3141 +▁kālele -3142 +▁kūpuna -3143 +▁master -3144 +▁native -3145 +▁pāpale -3146 +▁pōpoki -3147 +▁record -3148 +▁samuel -3149 +▁secret -3150 +▁shōnen -3151 +▁spider -3152 +▁tenrik -3153 +▁triple -3154 +▁wisdom -3155 +rotikova -3156 +▁antonio -3157 +▁collins -3158 +▁genesis -3159 +▁johnson -3160 +▁nolewai -3161 +▁project -3162 +▁republi -3163 +▁wozniak -3164 +aukekulia -3165 +omākaukau -3166 +▁hulahula -3167 +▁kānamapa -3168 +▁mikikana -3169 +▁wasasala -3170 +▁žalgiris -3171 +▁akāiuwapa -3172 +▁cantabria -3173 +▁kelekolio -3174 +▁meīkanasā -3175 +▁polynesian -3176 +▁washington -3177 +▁christopher -3178 +▁huapalapala -3179 +▁kauikeaouli -3180 +▁libertadores -3181 +▁lepupalikankie -3182 +," -3183 +,ʻ -3184 +ju -3185 +lb -3186 +mē -3187 +vė -3188 +κα -3189 +ен -3190 +ко -3191 +од -3192 +▁š -3193 +ahe -3194 +ddy -3195 +dwi -3196 +gal -3197 +gli -3198 +har -3199 +hip -3200 +iii -3201 +lly -3202 +maī -3203 +min -3204 +mpo -3205 +nai -3206 +ras -3207 +ray -3208 +sil -3209 +sse -3210 +tty -3211 +uin -3212 +upu -3213 +van -3214 +vėž -3215 +▁ -6863 +)" -6864 +/ˈ -6865 +fs -6866 +gt -6867 +hr -6868 +ht -6869 +ip -6870 +iv -6871 +kr -6872 +lp -6873 +lē -6874 +má -6875 +mã -6876 +pó -6877 +rō -6878 +sb -6879 +té -6880 +uc -6881 +zá -6882 +ém -6883 +öz -6884 +īe -6885 +ūō -6886 +βρ -6887 +ελ -6888 +θή -6889 +οδ -6890 +ου -6891 +οῖ -6892 +ύσ -6893 +ев -6894 +ел -6895 +ем -6896 +ес -6897 +жи -6898 +зы -6899 +пу -6900 +ть -6901 +اد -6902 +′‘ -6903 +▁= -6904 +▁» -6905 +▁ä -6906 +▁ø -6907 +▁ī -6908 +▁ż -6909 +▁α -6910 +▁ה -6911 +▁ب -6912 +▁ل -6913 +▁„ -6914 +%). -6915 +) -7780 +< -7781 +– -7782 +í -7783 +% -7784 ++ -7785 +! -7786 +о -7787 +è -7788 +σ -7789 +а -7790 +ã -7791 +[ -7792 +] -7793 +ç -7794 +? -7795 +и -7796 +е -7797 +| -7798 +α -7799 +н -7800 +’ -7801 +ñ -7802 +ä -7803 +т -7804 +ν -7805 +р -7806 +ο -7807 +† -7808 +с -7809 +ú -7810 +в -7811 +ρ -7812 +& -7813 +ô -7814 +τ -7815 +к -7816 +ε -7817 +ˈ -7818 +л -7819 +ü -7820 +$ -7821 +м -7822 +ö -7823 +я -7824 +ê -7825 +ž -7826 +η -7827 +` -7828 +# -7829 +ι -7830 +μ -7831 +* -7832 +π -7833 +° -7834 +п -7835 +у -7836 +κ -7837 +λ -7838 +ό -7839 +ː -7840 +д -7841 +â -7842 +г -7843 +і -7844 +ا -7845 += -7846 +š -7847 +ɪ -7848 +γ -7849 +δ -7850 +б -7851 +з -7852 +י -7853 +~ -7854 +ı -7855 +å -7856 +ー -7857 +ɑ -7858 +ά -7859 +ί -7860 +ύ -7861 +õ -7862 +ł -7863 +ɔ -7864 +ή -7865 +ἡ -7866 +の -7867 +ė -7868 +ǀ -7869 +ə -7870 +θ -7871 +ו -7872 +ר -7873 +_ -7874 +ɛ -7875 +ω -7876 +й -7877 +ь -7878 +ה -7879 +“ -7880 +υ -7881 +ж -7882 +ر -7883 +” -7884 +έ -7885 +β -7886 +ы -7887 +′ -7888 +à -7889 +د -7890 +ل -7891 +い -7892 +う -7893 +ë -7894 +ğ -7895 +ɐ -7896 +φ -7897 +ב -7898 +ب -7899 +ي -7900 +ἰ -7901 +ῖ -7902 +ß -7903 +ś -7904 +ǂ -7905 +ц -7906 +ї -7907 +א -7908 +ל -7909 +מ -7910 +ن -7911 +ḗ -7912 +ἐ -7913 +ὶ -7914 +こ -7915 +は -7916 +× -7917 +î -7918 +ï -7919 +ø -7920 +đ -7921 +ʊ -7922 +ш -7923 +щ -7924 +ְ -7925 +ת -7926 +ὸ -7927 +ῶ -7928 +• -7929 +銀 -7930 +ý -7931 +ă -7932 +č -7933 +İ -7934 +ŋ -7935 +ʁ -7936 +ʔ -7937 +ʘ -7938 +χ -7939 +ד -7940 +ע -7941 +و -7942 +ἀ -7943 +ἄ -7944 +⠸ -7945 +ん -7946 +ス -7947 +世 -7948 +座 -7949 +æ -7950 +ő -7951 +ş -7952 +ż -7953 +ǃ -7954 +ʃ -7955 +ώ -7956 +ф -7957 +ִ -7958 +ש -7959 +س -7960 +ه -7961 +ṓ -7962 +ả -7963 +ồ -7964 +ὐ -7965 +ῦ -7966 +„ -7967 +ま -7968 +も -7969 +れ -7970 +コ -7971 +マ -7972 +ミ -7973 +ラ -7974 +ン -7975 +・ -7976 +一 -7977 +國 -7978 +天 -7979 +子 -7980 +한 -7981 +÷ -7982 +ù -7983 +ć -7984 +ų -7985 +ư -7986 +ɡ -7987 +ɾ -7988 +̩ -7989 +ξ -7990 +ψ -7991 +х -7992 +ч -7993 +ָ -7994 +ּ -7995