{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "([bos])", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "([eos])", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "([unk])", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "([pad])", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "([mask])", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Metaspace", "replacement": "▁", "add_prefix_space": true }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "([eos])", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "([eos])", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "([eos])", "type_id": 1 } } ], "special_tokens": { "([bos])": { "id": "([bos])", "ids": [ 0 ], "tokens": [ "([bos])" ] }, "([eos])": { "id": "([eos])", "ids": [ 1 ], "tokens": [ "([eos])" ] } } }, "decoder": { "type": "Metaspace", "replacement": "▁", "add_prefix_space": true }, "model": { "type": "Unigram", "unk_id": 2, "vocab": [ [ "([bos])", 0.0 ], [ "([eos])", 0.0 ], [ "([unk])", 0.0 ], [ "([pad])", 0.0 ], [ "([mask])", 0.0 ], [ "▁", -2.59281702408172 ], [ "s", -2.846832096965846 ], [ "a", -3.355027224273531 ], [ "t", -3.393961431042454 ], [ "i", -3.4084006872371155 ], [ "e", -3.4593687402770925 ], [ "▁the", -3.809364677119774 ], [ "unk", -3.850929189246447 ], [ "]", -3.850965409526383 ], [ ")", -3.852990425372683 ], [ "▁([", -3.85299042537269 ], [ "r", -4.094359852161929 ], [ "d", -4.11316992845574 ], [ "o", -4.201950348043347 ], [ "▁and", -4.229721028930884 ], [ "▁to", -4.248849013980928 ], [ "n", -4.348259390081129 ], [ "▁of", -4.418050097301856 ], [ "ing", -4.468362591583357 ], [ "▁is", -4.477151270303802 ], [ "l", -4.494056484027906 ], [ "▁in", -4.575750056745386 ], [ "p", -4.584899128833191 ], [ "y", -4.637799480076254 ], [ "c", -4.644663089525897 ], [ "▁that", -4.6603864147085226 ], [ "u", -4.783309159320428 ], [ "ed", -4.846814524751098 ], [ "▁we", -4.880079560393588 ], [ "▁it", -4.893225004775269 ], [ "m", -4.919627253351287 ], [ "▁you", -4.985103298055368 ], [ "f", -5.027307044393842 ], [ "g", -5.171265598312889 ], [ "▁be", -5.249873992154958 ], [ "k", -5.306694390961949 ], [ "▁thi", -5.332036394347231 ], [ "al", -5.358246528230415 ], [ "w", -5.3757230492858845 ], [ "h", -5.380264963144688 ], [ "▁so", -5.387409655248286 ], [ "▁are", -5.400093862250188 ], [ "or", -5.419532476251575 ], [ "b", -5.432614155894891 ], [ "ly", -5.435895869014663 ], [ "▁not", -5.4665966086609945 ], [ "v", -5.509818024110549 ], [ "in", -5.519912000688162 ], [ "an", -5.595795502816262 ], [ "▁have", -5.60883245933184 ], [ "on", -5.6256645337389575 ], [ "▁for", -5.635595512460842 ], [ "▁do", -5.6374099852425505 ], [ "▁was", -5.723276308134322 ], [ "▁on", -5.743095684018304 ], [ "le", -5.76070836248679 ], [ "re", -5.77655479919024 ], [ "▁they", -5.795448251103968 ], [ "er", -5.7992845792341114 ], [ "ch", -5.81478145540922 ], [ "▁re", -5.843487125026712 ], [ "▁with", -5.8874500113813735 ], [ "ic", -5.918799565838919 ], [ "▁but", -5.930793316234986 ], [ "▁what", -5.932288213721186 ], [ "▁can", -5.953510754321824 ], [ "ur", -5.968178990473417 ], [ "ter", -5.968256487230722 ], [ "▁as", -5.972678693374657 ], [ "ce", -6.029926315071711 ], [ "▁me", -6.046376462967448 ], [ "ent", -6.050262468851077 ], [ "th", -6.067805528127913 ], [ "▁ma", -6.131960747326147 ], [ "▁at", -6.138386657031125 ], [ "about", -6.167158251053305 ], [ "un", -6.181842915931661 ], [ "▁all", -6.20424986580228 ], [ "▁us", -6.217295978939266 ], [ "ve", -6.225380529297736 ], [ "▁there", -6.232464937756388 ], [ "▁my", -6.23614114257248 ], [ "▁an", -6.3069852979408 ], [ "ation", -6.3190710988219045 ], [ "▁he", -6.323972547255821 ], [ "▁like", -6.384587060094013 ], [ "▁people", -6.387702184589152 ], [ "▁co", -6.4197126557699224 ], [ "▁one", -6.427689896795764 ], [ "▁our", -6.438420746463613 ], [ "▁if", -6.463213249855276 ], [ "ies", -6.466810786198042 ], [ "▁from", -6.470647565430909 ], [ "▁mo", -6.477409720708605 ], [ "▁con", -6.505790805736119 ], [ "▁or", -6.550931300395687 ], [ "0", -6.56122661435958 ], [ "nd", -6.5668699457644255 ], [ "▁out", -6.5684221767234785 ], [ "▁just", -6.5686010025050425 ], [ "ge", -6.586605784866393 ], [ "cause", -6.588241811302158 ], [ "▁see", -6.5949530396895994 ], [ "▁when", -6.599974572805861 ], [ "▁would", -6.606921677824124 ], [ "▁ex", -6.61041107046502 ], [ "▁tr", -6.618373395542154 ], [ "▁these", -6.62260645328524 ], [ "▁now", -6.632161104955264 ], [ "▁who", -6.635832866112693 ], [ "▁thing", -6.67135143948857 ], [ "▁know", -6.673675896544896 ], [ "▁up", -6.69640943895293 ], [ "tion", -6.707979332616 ], [ "me", -6.709936696310173 ], [ "▁go", -6.710623324221499 ], [ "▁think", -6.714133588775203 ], [ "▁pa", -6.71714944035719 ], [ "▁se", -6.718680072381458 ], [ "▁will", -6.720109781560021 ], [ "▁ho", -6.720170661330384 ], [ "▁by", -6.721095141355006 ], [ "▁di", -6.724710312743682 ], [ "▁how", -6.7254939284453705 ], [ "ck", -6.7313580469977214 ], [ "very", -6.741095758032021 ], [ "▁get", -6.746454814680545 ], [ "ll", -6.760221826176743 ], [ "▁pro", -6.763809825449721 ], [ "▁no", -6.770926703803614 ], [ "▁had", -6.7796436687805155 ], [ "▁ba", -6.786786701804624 ], [ "▁am", -6.786822442257263 ], [ "ment", -6.792776940927245 ], [ "▁more", -6.8025374347670695 ], [ "▁year", -6.812012015308449 ], [ "▁them", -6.817024115499338 ], [ "ide", -6.820768705286692 ], [ "▁some", -6.8293849455715545 ], [ "ity", -6.830203222215266 ], [ "▁going", -6.830635614280581 ], [ "▁time", -6.831841717883933 ], [ "1", -6.847393562136933 ], [ "ther", -6.857068158548849 ], [ "▁bo", -6.857135856112599 ], [ "▁were", -6.863668924034885 ], [ "▁their", -6.8645372292006765 ], [ "ive", -6.869128207203495 ], [ "▁mu", -6.886402777433895 ], [ "x", -6.904422101224576 ], [ "▁your", -6.911538672946254 ], [ "▁look", -6.918280672105752 ], [ "▁which", -6.933308643308452 ], [ "▁work", -6.9381661812620745 ], [ "▁2", -6.9469384737839395 ], [ "▁want", -6.963014386886217 ], [ "▁really", -6.974232818616819 ], [ "▁mi", -6.9811388765277975 ], [ "▁po", -6.989428952910444 ], [ "▁has", -6.99573133490755 ], [ "▁world", -6.9964067060377655 ], [ "▁br", -6.9987555818015785 ], [ "▁way", -6.999734719128316 ], [ "▁here", -7.019074310175704 ], [ "▁ca", -7.019578211637391 ], [ "us", -7.0243519005443495 ], [ "age", -7.041846360171347 ], [ "z", -7.05079536838384 ], [ "able", -7.062413259930416 ], [ "▁other", -7.069336328826822 ], [ "ng", -7.0740562743779325 ], [ "▁ha", -7.083605870276065 ], [ "▁could", -7.091024151027586 ], [ "▁make", -7.095302988139583 ], [ "▁la", -7.10003792423003 ], [ "one", -7.117391704778116 ], [ "▁actual", -7.128473711477547 ], [ "ver", -7.133692396499471 ], [ "▁into", -7.138511341814535 ], [ "tic", -7.139363297407975 ], [ "ally", -7.144123905557578 ], [ "ers", -7.145039705434364 ], [ "▁where", -7.148431169833696 ], [ "▁fe", -7.162143170057698 ], [ "▁than", -7.168255166847715 ], [ "am", -7.194024878731092 ], [ "▁did", -7.200844896395903 ], [ "▁she", -7.214015726235695 ], [ "ugh", -7.215069305179371 ], [ "ous", -7.225648440582878 ], [ "les", -7.227535434909608 ], [ "▁j", -7.227701515834608 ], [ "5", -7.22838724161401 ], [ "▁pre", -7.228412231025503 ], [ "▁say", -7.238257056896082 ], [ "▁cl", -7.245947147236425 ], [ "▁vi", -7.250369116702121 ], [ "▁li", -7.267006788594745 ], [ "▁new", -7.271103757230721 ], [ "▁been", -7.273765287272704 ], [ "um", -7.289790697831917 ], [ "▁any", -7.291267592684676 ], [ "▁bi", -7.295080757654635 ], [ "▁his", -7.299760201510503 ], [ "▁sp", -7.305303730474352 ], [ "ical", -7.307323351984438 ], [ "hose", -7.3114913033516125 ], [ "▁need", -7.313058253270235 ], [ "▁right", -7.322625855860206 ], [ "▁comp", -7.323471947989754 ], [ "▁take", -7.326942026217436 ], [ "▁even", -7.327454700703431 ], [ "▁over", -7.329218759587572 ], [ "▁start", -7.339334478402948 ], [ "▁3", -7.341330209983386 ], [ "▁car", -7.354030933819425 ], [ "▁lo", -7.362442350524409 ], [ "com", -7.374098833333429 ], [ "▁well", -7.390161285883304 ], [ "▁something", -7.390165545491165 ], [ "▁every", -7.390369002934392 ], [ "▁ar", -7.413232701641375 ], [ "ize", -7.426412981313161 ], [ "▁back", -7.427173568621068 ], [ "▁10", -7.441486980314046 ], [ "per", -7.445071842052444 ], [ "▁cr", -7.456267263418599 ], [ "war", -7.457041121613809 ], [ "▁fr", -7.459029556799015 ], [ "▁call", -7.46112374109263 ], [ "▁most", -7.462118302109969 ], [ "▁pr", -7.465632258952185 ], [ "▁also", -7.465693588901637 ], [ "1s", -7.4673436351197235 ], [ "j", -7.470293333180523 ], [ "▁sai", -7.47782202687231 ], [ "000", -7.483070644854715 ], [ "▁life", -7.484842783513717 ], [ "ful", -7.487346680602727 ], [ "ance", -7.502415813556965 ], [ "▁ga", -7.505566986745112 ], [ "▁come", -7.507829702035803 ], [ "▁kind", -7.509654685818351 ], [ "ence", -7.530438073529829 ], [ "▁little", -7.547114481715971 ], [ "▁lot", -7.55188953364898 ], [ "▁talk", -7.5589159495899825 ], [ "▁part", -7.561863039692147 ], [ "ction", -7.565679549641631 ], [ "▁happen", -7.57815173458663 ], [ "▁many", -7.58130503159693 ], [ "▁her", -7.596658925219376 ], [ "4", -7.607147336680983 ], [ "each", -7.612947571030476 ], [ "▁mean", -7.619962514213633 ], [ "▁creat", -7.661761922798204 ], [ "▁different", -7.67122410069279 ], [ "▁let", -7.677441531038708 ], [ "▁human", -7.680394078987549 ], [ "day", -7.685829986670678 ], [ "00000", -7.687110569454335 ], [ "▁change", -7.688074055024863 ], [ "▁show", -7.705816323646177 ], [ "around", -7.710025462854299 ], [ "▁good", -7.714050133917926 ], [ "▁does", -7.715047479814897 ], [ "▁through", -7.718038148789979 ], [ "▁fl", -7.7183657794136735 ], [ "▁bu", -7.718689852896475 ], [ "▁real", -7.721158921371002 ], [ "q", -7.722948646486868 ], [ "▁down", -7.731549014813332 ], [ "▁why", -7.749744153065748 ], [ "▁live", -7.757995308103174 ], [ "ated", -7.769146541041033 ], [ "▁tell", -7.772980291921325 ], [ "▁idea", -7.77749698230712 ], [ "self", -7.79441466402 ], [ "▁same", -7.806529851170208 ], [ "▁give", -7.808789063030774 ], [ "ture", -7.816124821978905 ], [ "▁gu", -7.822907720829415 ], [ "▁dec", -7.827497442862535 ], [ "▁sha", -7.831994042718744 ], [ "▁cha", -7.835305423960509 ], [ "rate", -7.840925431144495 ], [ "▁problem", -7.859746127968331 ], [ "log", -7.867267186770574 ], [ "▁fact", -7.876315750422663 ], [ "9", -7.8859793746573 ], [ "▁big", -7.900866055217808 ], [ "par", -7.902588172614095 ], [ "▁great", -7.907033482030775 ], [ "▁app", -7.91427941067953 ], [ "▁find", -7.916187196447025 ], [ "▁after", -7.940753458457673 ], [ "▁system", -7.943278029810768 ], [ "▁place", -7.94700632655662 ], [ "▁pu", -7.947172864717192 ], [ "▁gra", -7.964333957878741 ], [ "▁countr", -7.970206863296095 ], [ "▁build", -7.976780126354955 ], [ "▁20", -7.978013543613265 ], [ "▁hear", -7.978433112810803 ], [ "▁again", -7.989773163596327 ], [ "6", -7.990880109971288 ], [ "▁imp", -7.991637432829929 ], [ "▁learn", -7.9959131203456355 ], [ "form", -7.999593417984542 ], [ "▁own", -8.017045352055355 ], [ "▁gene", -8.021482294420894 ], [ "▁long", -8.024830416011637 ], [ "▁made", -8.025958739187063 ], [ "▁qu", -8.029481112958162 ], [ "light", -8.042944650519123 ], [ "▁question", -8.04733201088698 ], [ "line", -8.048098265300471 ], [ "▁should", -8.05086940090139 ], [ "▁came", -8.062749375963078 ], [ "ness", -8.075393218062162 ], [ "8", -8.080287699177687 ], [ "▁end", -8.087216166105685 ], [ "▁feel", -8.089768108480714 ], [ "▁turn", -8.097471494540764 ], [ "▁person", -8.098872045094318 ], [ "▁technolog", -8.100599214400695 ], [ "%", -8.122602302046138 ], [ "▁hu", -8.126737314662325 ], [ "▁design", -8.12674245324745 ], [ "▁help", -8.130318117974053 ], [ "▁brain", -8.138463246691146 ], [ "▁last", -8.142073157234577 ], [ "▁important", -8.147052636620238 ], [ "▁before", -8.14837919317405 ], [ "▁high", -8.15195082403437 ], [ "▁never", -8.164694320286374 ], [ "▁thought", -8.164796046272262 ], [ "▁trans", -8.169287856512064 ], [ "▁him", -8.171411032689104 ], [ "▁might", -8.183200086027288 ], [ "7", -8.190653526557211 ], [ "▁understand", -8.19065373044177 ], [ "▁interest", -8.196802373595887 ], [ "▁power", -8.198573739588852 ], [ "▁better", -8.19988067880684 ], [ "land", -8.20388344699932 ], [ "▁found", -8.206082011890224 ], [ "▁play", -8.215114406919103 ], [ "▁still", -8.239390820997944 ], [ "▁fun", -8.239404594536715 ], [ "▁cit", -8.245195776069933 ], [ "▁point", -8.252267399561559 ], [ "▁school", -8.256383568070238 ], [ "▁together", -8.25829247621957 ], [ "▁old", -8.268629977562462 ], [ "▁example", -8.27323704876234 ], [ "▁next", -8.281402530476942 ], [ "wome", -8.282154732814078 ], [ "▁state", -8.29468188529561 ], [ "▁under", -8.305898457550091 ], [ "▁number", -8.310117772682384 ], [ "▁course", -8.321718517074732 ], [ "▁ago", -8.327507330243627 ], [ "▁water", -8.328825111687745 ], [ "▁data", -8.333203001750057 ], [ "▁grow", -8.340873398838944 ], [ "▁simpl", -8.349218229729566 ], [ "▁famil", -8.369363631544427 ], [ "tween", -8.37383437525906 ], [ "▁develop", -8.374643423007722 ], [ "que", -8.375200998662995 ], [ "2", -8.376102518707132 ], [ "▁america", -8.379573798445456 ], [ "▁believe", -8.394053582941464 ], [ "▁small", -8.415568961449141 ], [ "▁maybe", -8.42368192193646 ], [ "▁become", -8.424145125887913 ], [ "room", -8.441630177731476 ], [ "▁far", -8.444549208994184 ], [ "▁health", -8.467222838689395 ], [ "▁space", -8.471483420396034 ], [ "▁word", -8.478065675740668 ], [ "▁children", -8.499512226612513 ], [ "alway", -8.520587524981275 ], [ "▁reason", -8.526841785089601 ], [ "▁away", -8.544995251715026 ], [ "abilit", -8.550836871274203 ], [ "▁econom", -8.56406438391495 ], [ "▁experience", -8.567033977916246 ], [ "▁large", -8.57014429948284 ], [ "▁everything", -8.57959705440446 ], [ "▁friend", -8.58504421926433 ], [ "▁open", -8.591645742993963 ], [ "▁face", -8.607261919389757 ], [ "▁communit", -8.610078928970474 ], [ "▁computer", -8.61327278832886 ], [ "▁money", -8.620019706055368 ], [ "▁information", -8.623660752747348 ], [ "▁social", -8.647076269648476 ], [ "graph", -8.649930279809734 ], [ "▁writ", -8.660041940377742 ], [ "▁walk", -8.660537038157099 ], [ "▁sense", -8.661222653828128 ], [ "▁africa", -8.664304453839758 ], [ "▁picture", -8.666496894472793 ], [ "▁process", -8.674246917973406 ], [ "ready", -8.687949675721452 ], [ "▁group", -8.708420511827635 ], [ "▁future", -8.719915098813502 ], [ "▁elect", -8.721907126457397 ], [ "▁probabl", -8.743286352848367 ], [ "3", -8.748160451209834 ], [ "▁energ", -8.749211217327893 ], [ "▁imagine", -8.752781394092509 ], [ "▁science", -8.75997395003522 ], [ "▁food", -8.762990753474401 ], [ "▁connect", -8.77100174324766 ], [ "quite", -8.773647509239836 ], [ "▁government", -8.775722423993372 ], [ "▁answer", -8.78002900412545 ], [ "▁global", -8.780619828904241 ], [ "▁stand", -8.785036546626142 ], [ "▁keep", -8.788654740788271 ], [ "▁animal", -8.804862510265878 ], [ "where", -8.811834342327783 ], [ "▁possibl", -8.820056966136203 ], [ "across", -8.830962661034436 ], [ "▁control", -8.838737620869553 ], [ "▁research", -8.844606219767416 ], [ "▁product", -8.844911614902509 ], [ "▁stuff", -8.845260960850675 ], [ "▁remember", -8.845915483110893 ], [ "▁while", -8.85051297862519 ], [ "▁child", -8.853110416617389 ], [ "▁business", -8.859769930901034 ], [ "▁physic", -8.861164207607107 ], [ "▁environment", -8.873814996112682 ], [ "▁wonder", -8.873819960339489 ], [ "▁politic", -8.88738762226837 ], [ "▁student", -8.89767468539825 ], [ "▁public", -8.905987909835986 ], [ "▁societ", -8.929935600634728 ], [ "▁machine", -8.93135882460237 ], [ "special", -8.932997840222589 ], [ "▁particular", -8.952259164739559 ], [ "▁effect", -8.961776030509103 ], [ "▁disease", -8.961777244037055 ], [ "▁study", -8.972872315045484 ], [ "▁minute", -8.977346606956452 ], [ "▁discover", -9.00922662860049 ], [ "▁certain", -9.013088908038048 ], [ "▁figure", -9.021642649757036 ], [ "▁wrong", -9.023207113478072 ], [ "▁market", -9.031848417684849 ], [ "▁organiz", -9.034216957681174 ], [ "▁program", -9.04775239624157 ], [ "▁language", -9.063099143498777 ], [ "ground", -9.082829041793172 ], [ "▁education", -9.083659845011791 ], [ "▁incredibl", -9.089492419062424 ], [ "▁everybody", -9.097885260712856 ], [ "▁beautiful", -9.12263050089388 ], [ "▁challenge", -9.13305180006321 ], [ "▁individual", -9.136549811286418 ], [ "▁robot", -9.149778414703448 ], [ "▁difference", -9.166787774331064 ], [ "▁object", -9.1731284477078 ], [ "▁follow", -9.189619820239848 ], [ "▁success", -9.194249482061554 ], [ "▁somebody", -9.201701855207816 ], [ "▁collect", -9.205481082183333 ], [ "▁natural", -9.227272303426044 ], [ "▁scientist", -9.28756625454666 ], [ "▁opportunit", -9.292673515244624 ], [ "▁support", -9.306075784827351 ], [ "▁difficult", -9.31336747480424 ], [ "▁network", -9.314413501104752 ], [ "▁behavior", -9.345234221100345 ], [ "▁structure", -9.372588216236457 ], [ "▁relationship", -9.379266035832249 ], [ "▁communicat", -9.441462488984255 ], [ "▁absolute", -9.53481354000388 ], [ "▁neighbor", -9.567973071869298 ], [ "[", -10.029248769845 ], [ "+", -13.924195720156202 ], [ "£", -13.924295720156202 ], [ "€", -13.924395720156202 ], [ "$", -13.924495720156202 ], [ "(", -13.924595720156203 ], [ "-", -13.924695720156205 ], [ ".", -13.924795720156204 ], [ ">", -13.924895720156186 ], [ "¢", -13.924895720156202 ], [ "<", -13.924895720156202 ], [ "/", -13.924895720156202 ] ] } }