tokenizer / bilingual_sp.vocab
khulnasoft's picture
Upload folder using huggingface_hub
420a4ce verified
<pad> 0
<unk> 0
<s> 0
</s> 0
▁" -0
": -1
re -2
as -3
te -4
on -5
se -6
{" -7
▁{" -8
sc -9
tr -10
ore -11
score -12
"} -13
ue -14
pas -15
sed -16
ason -17
passed -18
reason -19
li -20
"}, -21
▁tr -22
▁true -23
.0 -24
.0, -25
ten -26
en -27
▁1 -28
▁1.0, -29
ty -30
lity -31
▁0 -32
ce -33
", -34
OK -35
ua -36
xt -37
▁( -38
cte -39
▁OK -40
tent -41
ontent -42
▁0. -43
ng -44
ad -45
ha -46
la -47
or -48
▁s -49
ab -50
le -51
pr -52
ra -53
ti -54
abi -55
len -56
qua -57
)"}, -58
cter -59
hara -60
lang -61
text -62
tion -63
adabi -64
tence -65
content -66
quality -67
haracter -68
adability -69
00 -70
▁0.8 -71
▁5 -72
bn -73
oo -74
7, -75
is -76
he -77
ni -78
al -79
am -80
ks -81
ou -82
xe -83
▁a -84
▁d -85
▁f -86
▁w -87
Mi -88
Re -89
av -90
bu -91
ca -92
ct -93
de -94
du -95
fi -96
ge -97
gt -98
ia -99
ne -100
nu -101
}} -102
▁< -103
)"} -104
.", -105
Sen -106
Too -107
Uni -108
amp -109
avg -110
che -111
cks -112
ctu -113
dup -114
gth -115
hor -116
ibu -117
num -118
opr -119
ord -120
our -121
ppr -122
que -123
sen -124
tru -125
txt -126
xed -127
▁50 -128
alse -129
cted -130
dete -131
file -132
hort -133
iate -134
istr -135
lica -136
line -137
samp -138
sour -139
uage -140
▁len -141
)"}}} -142
Mixed -143
cture -144
▁0.0, -145
▁0.7, -146
▁appr -147
▁stru -148
▁text -149
▁word -150
Unique -151
checks -152
length -153
sample -154
source -155
tences -156
▁distr -157
▁false -158
▁short -159
Content -160
duplica -161
ibution -162
opriate -163
▁50)"}, -164
Sentence -165
detected -166
language -167
▁content -168
Character -169
character -170
sentences -171
characters -172
▁structure -173
Readability -174
duplication -175
readability -176
▁appropriate -177
▁distribution -178
0000 -179
5, -180
▁(2 -181
55, -182
▁en -183
▁0.855, -184
▁4 -185
▁3 -186
54 -187
to -188
ve -189
▁bn -190
▁to -191
09 -192
14 -193
আম -194
00000000 -195
15 -196
28 -197
714 -198
▁3. -199
0909 -200
5454 -201
.0)"}}} -202
1, -203
30 -204
69 -205
▁প -206
285 -207
আমি -208
▁4. -209
285714 -210
5)"}}} -211
ry -212
▁e -213
▁ব -214
▁স -215
769 -216
▁5. -217
▁li -218
very -219
▁(24 -220
▁0.8, -221
00001, -222
▁0.815 -223
▁4.0)"}}} -224
0000000000001, -225
0, -226
17 -227
2, -228
34 -229
38 -230
46 -231
6, -232
8, -233
ay -234
go -235
lo -236
কর -237
তি -238
তে -239
দি -240
ন্ -241
য় -242
যা -243
লে -244
াম -245
ার -246
্র -247
▁m -248
▁আ -249
▁খ -250
230 -251
347 -252
hoo -253
করি -254
দিন -255
ন্দ -256
▁(3 -257
▁2, -258
▁3, -259
▁4, -260
▁6, -261
▁7, -262
▁8, -263
▁go -264
▁is -265
▁lo -266
▁sc -267
3846 -268
4347 -269
hool -270
আমার -271
্রতি -272
▁(21 -273
▁(22 -274
▁(25 -275
▁(26 -276
▁10, -277
▁করি -278
▁0.82 -279
▁love -280
230769 -281
3)"}}} -282
4)"}}} -283
6)"}}} -284
▁every -285
▁প্রতি -286
▁school -287
09090909 -288
54545454 -289
▁3.5)"}}} -290
▁প্রতিদিন -291
285714285714 -292
08 -293
26 -294
3, -295
39 -296
52 -297
9, -298
Ba -299
My -300
Ra -301
at -302
ci -303
er -304
es -305
hi -306
ic -307
in -308
ke -309
tb -310
us -311
কট -312
কা -313
কু -314
ছে -315
ছো -316
টব -317
বহ -318
বা -319
রহ -320
লো -321
সি -322
িম -323
ুব -324
েল -325
োন -326
▁b -327
▁n -328
▁p -329
▁y -330
▁ন -331
130 -332
154 -333
280 -334
652 -335
826 -336
The -337
all -338
ame -339
day -340
eat -341
esh -342
her -343
him -344
lad -345
lay -346
rci -347
ter -348
কটি -349
ছোট -350
টবল -351
বহা -352
য়া -353
যাই -354
়তে -355
ালো -356
্কু -357
্যা -358
▁1, -359
▁5, -360
▁9, -361
▁Ba -362
▁Ra -363
▁ha -364
▁in -365
▁ni -366
▁re -367
▁পছ -368
▁বই -369
▁রহ -370
▁সু -371
0869 -372
154, -373
1739 -374
nger -375
ning -376
ooks -377
ootb -378
sten -379
usic -380
কালে -381
ন্দর -382
বাসি -383
য়াম -384
ুটবল -385
েলতে -386
▁(14 -387
▁(15 -388
▁(17 -389
▁(23 -390
▁(31 -391
▁(32 -392
▁4.8 -393
▁5.2 -394
▁day -395
▁exe -396
▁mor -397
▁sis -398
▁you -399
▁আছে -400
▁খুব -401
▁ছোট -402
▁নাম -403
▁বোন -404
▁যাই -405
nglad -406
rcise -407
্কুলে -408
▁0.84 -409
▁have -410
▁like -411
▁live -412
▁name -413
▁nice -414
▁play -415
▁read -416
▁very -417
▁weat -418
▁আবহা -419
▁ব্যা -420
▁রহিম -421
08695, -422
09091, -423
153846 -424
303846 -425
43477, -426
▁0.824 -427
▁0.829 -428
▁Rahim -429
▁books -430
▁footb -431
▁music -432
▁today -433
▁খেলতে -434
▁পছন্দ -435
▁সকালে -436
1739130 -437
230769, -438
2804347 -439
545455, -440
ালোবাসি -441
▁0.8652 -442
▁0.8714 -443
▁0.8769 -444
▁listen -445
▁sister -446
▁সুন্দর -447
▁স্কুলে -448
ngladesh -449
▁morning -450
▁weather -451
▁younger -452
82608695, -453
▁3.0)"}}} -454
▁3.3)"}}} -455
▁3.4)"}}} -456
▁3.6)"}}} -457
▁4.3)"}}} -458
▁4.4)"}}} -459
▁4.5)"}}} -460
▁4.8)"}}} -461
▁5.0)"}}} -462
▁5.2)"}}} -463
▁5.5)"}}} -464
▁5.6)"}}} -465
▁exercise -466
▁football -467
▁ব্যায়াম -468
153846154, -469
▁0.8303846 -470
▁0.82804347 -471
▁Bangladesh -472
173913043477, -473
230769230769, -474
285714285714, -475
▁0.8454545454 -476
0909090909091, -477
2857142857143, -478
(1 -479
(2 -480
(3 -481
)" -482
." -483
.2 -484
.3 -485
.4 -486
.5 -487
.6 -488
.7 -489
.8 -490
0) -491
0. -492
01 -493
03 -494
04 -495
07 -496
1. -497
10 -498
13 -499
2) -500
21 -501
22 -502
23 -503
24 -504
25 -505
29 -506
3) -507
3. -508
31 -509
32 -510
4) -511
4, -512
4. -513
40 -514
42 -515
43 -516
45 -517
47 -518
5) -519
5. -520
50 -521
53 -522
55 -523
57 -524
6) -525
60 -526
61 -527
65 -528
71 -529
73 -530
76 -531
77 -532
78 -533
8) -534
80 -535
81 -536
82 -537
83 -538
84 -539
85 -540
86 -541
87 -542
90 -543
91 -544
92 -545
95 -546
Ch -547
Co -548
Se -549
Th -550
To -551
Un -552
ac -553
ag -554
ah -555
an -556
ap -557
ar -558
ba -559
bi -560
bo -561
ch -562
ck -563
co -564
da -565
di -566
ea -567
ec -568
ed -569
et -570
ev -571
ex -572
fa -573
fo -574
gl -575
gu -576
ho -577
ib -578
ik -579
il -580
im -581
io -582
iq -583
it -584
iv -585
ix -586
ll -587
ls -588
me -589
mo -590
mp -591
mu -592
na -593
nc -594
nt -595
od -596
ok -597
ol -598
op -599
ot -600
ov -601
pa -602
pl -603
pp -604
qu -605
rc -606
rd -607
ri -608
rn -609
ro -610
rs -611
rt -612
ru -613
sa -614
sh -615
si -616
so -617
ss -618
st -619
th -620
tu -621
tx -622
uc -623
um -624
un -625
up -626
ur -627
ut -628
vg -629
we -630
wo -631
yo -632
}, -633
আছ -634
আব -635
খু -636
খে -637
ছন -638
টি -639
দর -640
না -641
পছ -642
প্ -643
বই -644
বল -645
বো -646
ব্ -647
মা -648
মি -649
রত -650
রি -651
লত -652
সক -653
সু -654
স্ -655
হা -656
হি -657
়ত -658
়া -659
াই -660
ায -661
াল -662
াস -663
িদ -664
িন -665
ুট -666
ুন -667
ুল -668
োট -669
োব -670
্ক -671
্দ -672
্য -673
▁2 -674
▁6 -675
▁7 -676
▁8 -677
▁9 -678
▁B -679
▁O -680
▁R -681
▁c -682
▁g -683
▁h -684
▁i -685
▁l -686
▁r -687
▁t -688
▁v -689
▁{ -690
▁ক -691
▁ছ -692
▁য -693
▁র -694
"}} -695
.0) -696
0.0 -697
000 -698
001 -699
090 -700
091 -701
1.0 -702
14, -703
142 -704
143 -705
153 -706
173 -707
217 -708
3.0 -709
303 -710
304 -711
307 -712
384 -713
391 -714
4.0 -715
409 -716
434 -717
454 -718
461 -719
5.0 -720
500 -721
54, -722
545 -723
608 -724
615 -725
69, -726
692 -727
77, -728
869 -729
928 -730
95, -731
Cha -732
Con -733
Rah -734
ada -735
ade -736
ali -737
ang -738
aso -739
ass -740
ate -741
ati -742
bal -743
boo -744
ces -745
cha -746
con -747
dis -748
ead -749
eng -750
ete -751
eve -752
exe -753
fal -754
foo -755
gua -756
har -757
hav -758
hec -759
ice -760
ile -761
ili -762
ing -763
ion -764
ise -765
ixe -766
lic -767
lik -768
lin -769
lis -770
lit -771
liv -772
lse -773
mor -774
nam -775
nge -776
ngl -777
niq -778
nte -779
ook -780
ool -781
oot -782
orn -783
ort -784
ove -785
pla -786
ple -787
pli -788
pri -789
pro -790
rce -791
rea -792
sam -793
sch -794
sco -795
sic -796
sou -797
ste -798
str -799
tec -800
ted -801
tex -802
the -803
tod -804
tri -805
uag -806
uct -807
ung -808
ure -809
uti -810
ver -811
wor -812
xer -813
you -814
আমা -815
ছন্ -816
তিদ -817
নাম -818
বাস -819
রতি -820
রহি -821
লতে -822
ায় -823
ালে -824
ুটব -825
ুন্ -826
ুলে -827
োবা -828
▁(1 -829
▁10 -830
▁ap -831
▁da -832
▁mu -833
▁sh -834
▁si -835
▁te -836
▁ve -837
▁we -838
▁আছ -839
▁আব -840
▁কর -841
▁খু -842
▁খে -843
▁ছো -844
▁প্ -845
▁বো -846
▁ব্ -847
▁যা -848
▁সক -849
▁স্ -850
"}}} -851
0.0, -852
1.0, -853
1428 -854
1538 -855
2)"} -856
2608 -857
3)"} -858
3038 -859
3478 -860
4)"} -861
455, -862
4615 -863
5)"} -864
5217 -865
5714 -866
6)"} -867
695, -868
714, -869
7143 -870
769, -871
7692 -872
8)"} -873
9285 -874
Bang -875
Mixe -876
Rahi -877
Uniq -878
acte -879
adab -880
ades -881
asse -882
athe -883
chec -884
cise -885
have -886
iste -887
leng -888
live -889
love -890
ngua -891
nice -892
nten -893
onte -894
orni -895
oung -896
pria -897
read -898
reas -899
tbal -900
tenc -901
true -902
uctu -903
কুলে -904
ছন্দ -905
তিদি -906
যায় -907
রহিম -908
লোবা -909
ুন্দ -910
▁0.7 -911
▁3.0 -912
▁3.3 -913
▁3.4 -914
▁3.5 -915
▁3.6 -916
" -917
-918
e -919
s -920
t -921
: -922
r -923
a -924
n -925
o -926
, -927
c -928
i -929
u -930
d -931
l -932
0 -933
p -934
. -935
{ -936
} -937
1 -938
h -939
g -940
_ -941
y -942
b -943
x -944
5 -945
( -946
) -947
K -948
O -949
q -950
m -951
8 -952
4 -953
f -954
C -955
7 -956
2 -957
v -958
3 -959
k -960
M -961
R -962
T -963
w -964
< -965
S -966
U -967
6 -968
9 -969
ি -970
-971
-972
-973
I -974
-975
-976
-977
-978
-979
-980
-981
-982
-983
-984
-985
-986
-987
-988
-989
-990
-991
-992
-993
-994
B -995