Upload tokenizer

tokenizer.json

@@ -217,40 +217,40 @@
       "|": 69,
       "}": 70,
       "~": 71,
-      "##u": 72,
-      "##m": 73,
-      "##p": 74,
-      "##i": 75,
-      "##e": 76,
-      "##a": 77,
-      "##l": 78,
-      "##d": 79,
-      "##o": 80,
-      "##n": 81,
-      "##g": 82,
-      "##h": 83,
-      "##t": 84,
-      "##0": 85,
-      "##5": 86,
-      "##r": 87,
-      "##s": 88,
-      "##w": 89,
-      "##y": 90,
-      "##f": 91,
-      "##x": 92,
-      "##c": 93,
-      "##z": 94,
-      "##9": 95,
-      "##4": 96,
-      "##8": 97,
-      "##1": 98,
-      "##b": 99,
+      "##e": 72,
+      "##r": 73,
+      "##t": 74,
+      "##a": 75,
+      "##n": 76,
+      "##i": 77,
+      "##g": 78,
+      "##s": 79,
+      "##d": 80,
+      "##o": 81,
+      "##m": 82,
+      "##b": 83,
+      "##p": 84,
+      "##u": 85,
+      "##l": 86,
+      "##c": 87,
+      "##y": 88,
+      "##z": 89,
+      "##h": 90,
+      "##w": 91,
+      "##f": 92,
+      "##k": 93,
+      "##4": 94,
+      "##j": 95,
+      "##v": 96,
+      "##0": 97,
+      "##8": 98,
+      "##1": 99,
       "##q": 100,
-      "##2": 101,
-      "##v": 102,
-      "##k": 103,
-      "##3": 104,
-      "##j": 105,
+      "##x": 101,
+      "##3": 102,
+      "##5": 103,
+      "##2": 104,
+      "##9": 105,
       "##6": 106,
       "##7": 107,
       "##he": 108,
@@ -9740,8 +9740,8 @@
       "simpl": 9592,
       "thrusting": 9593,
       "intact": 9594,
-      "##phe": 9595,
-      "##rus": 9596,
+      "##rus": 9595,
+      "##phe": 9596,
       "cellar": 9597,
       "borrow": 9598,
       "tomb": 9599,
@@ -21890,8 +21890,8 @@
       "mellow": 21742,
       "recesses": 21743,
       "nondescript": 21744,
-      "##oom": 21745,
-      "##nuts": 21746,
+      "##nuts": 21745,
+      "##oom": 21746,
       "##idences": 21747,
       "altering": 21748,
       "ordinarily": 21749,
@@ -23289,8 +23289,8 @@
       "unwitting": 23141,
       "ogling": 23142,
       "horrendous": 23143,
-      "##uzz": 23144,
-      "##din": 23145,
+      "##din": 23144,
+      "##uzz": 23145,
       "undressing": 23146,
       "overdue": 23147,
       "trask": 23148,
@@ -25866,8 +25866,8 @@
       "casserole": 25718,
       "resilient": 25719,
       "latham": 25720,
-      "##pace": 25721,
-      "##grove": 25722,
+      "##grove": 25721,
+      "##pace": 25722,
       "watt": 25723,
       "lightness": 25724,
       "visage": 25725,
@@ -26792,9 +26792,9 @@
       "demolished": 26644,
       "jus": 26645,
       "zem": 26646,
-      "##mies": 26647,
-      "##eep": 26648,
-      "##tus": 26649,
+      "##eep": 26647,
+      "##tus": 26648,
+      "##mies": 26649,
       "blub": 26650,
       "talbot": 26651,
       "departments": 26652,
@@ -28185,8 +28185,8 @@
       "governess": 28037,
       "implacable": 28038,
       "vampaneze": 28039,
-      "##ugged": 28040,
-      "##rically": 28041,
+      "##rically": 28040,
+      "##ugged": 28041,
       "desari": 28042,
       "drax": 28043,
       "machi": 28044,
@@ -29352,8 +29352,8 @@
       "hinged": 29204,
       "nile": 29205,
       "##glasses": 29206,
-      "##has": 29207,
-      "##yran": 29208,
+      "##yran": 29207,
+      "##has": 29208,
       "stumped": 29209,
       "carrick": 29210,
       "##utha": 29211,
@@ -29611,8 +29611,8 @@
       "pleasured": 29463,
       "intermittent": 29464,
       "unscrewed": 29465,
-      "##pread": 29466,
-      "##nick": 29467,
+      "##nick": 29466,
+      "##pread": 29467,
       "##oreal": 29468,
       "disheart": 29469,
       "emban": 29470,
@@ -29845,8 +29845,8 @@
       "pf": 29697,
       "siber": 29698,
       "vie": 29699,
-      "##hinking": 29700,
-      "##sin": 29701,
+      "##sin": 29700,
+      "##hinking": 29701,
       "doomb": 29702,
       "atro": 29703,
       "insom": 29704,
@@ -30016,8 +30016,8 @@
       "imitate": 29868,
       "drinkers": 29869,
       "ethans": 29870,
-      "##uke": 29871,
-      "##ritch": 29872,
+      "##ritch": 29871,
+      "##uke": 29872,
       "spiel": 29873,
       "vehem": 29874,
       "besie": 29875,

tokenizer_config.json

@@ -1,11 +1,11 @@
 {
+  "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
   "do_lower_case": true,
   "mask_token": "[MASK]",
-  "model_max_length": 64,
+  "model_max_length": 48,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "special_tokens_map_file": null,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",

vocab.txt

@@ -70,40 +70,40 @@ z
 |
 }
 ~
-##u
-##m
-##p
-##i
 ##e
+##r
+##t
 ##a
-##l
-##d
-##o
 ##n
+##i
 ##g
-##h
-##t
-##0
-##5
-##r
 ##s
-##w
-##y
-##f
-##x
+##d
+##o
+##m
+##b
+##p
+##u
+##l
 ##c
+##y
 ##z
-##9
+##h
+##w
+##f
+##k
 ##4
+##j
+##v
+##0
 ##8
 ##1
-##b
 ##q
-##2
-##v
-##k
+##x
 ##3
-##j
+##5
+##2
+##9
 ##6
 ##7
 ##he
@@ -9593,8 +9593,8 @@ meals
 simpl
 thrusting
 intact
-##phe
 ##rus
+##phe
 cellar
 borrow
 tomb
@@ -21743,8 +21743,8 @@ prophecies
 mellow
 recesses
 nondescript
-##oom
 ##nuts
+##oom
 ##idences
 altering
 ordinarily
@@ -23142,8 +23142,8 @@ mounts
 unwitting
 ogling
 horrendous
-##uzz
 ##din
+##uzz
 undressing
 overdue
 trask
@@ -25719,8 +25719,8 @@ rivulets
 casserole
 resilient
 latham
-##pace
 ##grove
+##pace
 watt
 lightness
 visage
@@ -26645,9 +26645,9 @@ impersonal
 demolished
 jus
 zem
-##mies
 ##eep
 ##tus
+##mies
 blub
 talbot
 departments
@@ -28038,8 +28038,8 @@ dabbing
 governess
 implacable
 vampaneze
-##ugged
 ##rically
+##ugged
 desari
 drax
 machi
@@ -29205,8 +29205,8 @@ recycling
 hinged
 nile
 ##glasses
-##has
 ##yran
+##has
 stumped
 carrick
 ##utha
@@ -29464,8 +29464,8 @@ refocus
 pleasured
 intermittent
 unscrewed
-##pread
 ##nick
+##pread
 ##oreal
 disheart
 emban
@@ -29698,8 +29698,8 @@ alder
 pf
 siber
 vie
-##hinking
 ##sin
+##hinking
 doomb
 atro
 insom
@@ -29869,8 +29869,8 @@ aggie
 imitate
 drinkers
 ethans
-##uke
 ##ritch
+##uke
 spiel
 vehem
 besie