| { |
| "add_bos_token": true, |
| "add_eos_token": false, |
| "added_tokens_decoder": { |
| "0": { |
| "content": "<unk>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "1": { |
| "content": "<s>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "2": { |
| "content": "</s>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92352": { |
| "content": "Ｅ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92353": { |
| "content": "Ｆ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92354": { |
| "content": "Ｇ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92355": { |
| "content": "Ｈ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92356": { |
| "content": "Ｉ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92357": { |
| "content": "Ｊ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92358": { |
| "content": "Ｋ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92359": { |
| "content": "Ｌ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92360": { |
| "content": "Ｍ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92361": { |
| "content": "Ｎ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92362": { |
| "content": "Ｒ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92363": { |
| "content": "Ｕ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92364": { |
| "content": "Ｖ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92365": { |
| "content": "Ｗ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92366": { |
| "content": "Ｘ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92367": { |
| "content": "Ｙ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92368": { |
| "content": "Ｚ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92369": { |
| "content": "ａ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92370": { |
| "content": "ｂ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92371": { |
| "content": "ｃ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92372": { |
| "content": "ｄ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92373": { |
| "content": "ｅ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92374": { |
| "content": "ｆ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92375": { |
| "content": "ｇ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92376": { |
| "content": "ｈ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92377": { |
| "content": "ｉ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92378": { |
| "content": "ｊ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92379": { |
| "content": "ｋ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92380": { |
| "content": "ｌ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92381": { |
| "content": "ｍ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92382": { |
| "content": "ｎ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92383": { |
| "content": "ｏ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92384": { |
| "content": "ｐ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92385": { |
| "content": "ｑ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92386": { |
| "content": "ｒ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92387": { |
| "content": "ｓ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92388": { |
| "content": "ｔ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92389": { |
| "content": "ｕ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92390": { |
| "content": "ｖ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92391": { |
| "content": "ｗ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92392": { |
| "content": "ｘ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92393": { |
| "content": "ｙ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92394": { |
| "content": "ｚ", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92395": { |
| "content": "——", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92396": { |
| "content": "……", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92397": { |
| "content": "[UNUSED_TOKEN_0]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92398": { |
| "content": "[UNUSED_TOKEN_1]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92399": { |
| "content": "[UNUSED_TOKEN_2]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92400": { |
| "content": "[UNUSED_TOKEN_3]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92401": { |
| "content": "[UNUSED_TOKEN_4]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92402": { |
| "content": "[UNUSED_TOKEN_5]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92403": { |
| "content": "[UNUSED_TOKEN_6]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92404": { |
| "content": "[UNUSED_TOKEN_7]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92405": { |
| "content": "[UNUSED_TOKEN_8]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92406": { |
| "content": "[UNUSED_TOKEN_9]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92407": { |
| "content": "[UNUSED_TOKEN_10]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92408": { |
| "content": "[UNUSED_TOKEN_11]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92409": { |
| "content": "[UNUSED_TOKEN_12]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92410": { |
| "content": "[UNUSED_TOKEN_13]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92411": { |
| "content": "[UNUSED_TOKEN_14]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92412": { |
| "content": "[UNUSED_TOKEN_15]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92413": { |
| "content": "[UNUSED_TOKEN_16]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92414": { |
| "content": "[UNUSED_TOKEN_17]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92415": { |
| "content": "[UNUSED_TOKEN_18]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92416": { |
| "content": "[UNUSED_TOKEN_19]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92417": { |
| "content": "[UNUSED_TOKEN_20]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92418": { |
| "content": "[UNUSED_TOKEN_21]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92419": { |
| "content": "[UNUSED_TOKEN_22]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92420": { |
| "content": "[UNUSED_TOKEN_23]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92421": { |
| "content": "[UNUSED_TOKEN_24]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92422": { |
| "content": "[UNUSED_TOKEN_25]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92423": { |
| "content": "[UNUSED_TOKEN_26]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92424": { |
| "content": "[UNUSED_TOKEN_27]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92425": { |
| "content": "[UNUSED_TOKEN_28]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92426": { |
| "content": "[UNUSED_TOKEN_29]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92427": { |
| "content": "[UNUSED_TOKEN_30]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92428": { |
| "content": "[UNUSED_TOKEN_31]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92429": { |
| "content": "[UNUSED_TOKEN_32]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92430": { |
| "content": "[UNUSED_TOKEN_33]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92431": { |
| "content": "[UNUSED_TOKEN_34]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92432": { |
| "content": "[UNUSED_TOKEN_35]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92433": { |
| "content": "[UNUSED_TOKEN_36]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92434": { |
| "content": "[UNUSED_TOKEN_37]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92435": { |
| "content": "[UNUSED_TOKEN_38]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92436": { |
| "content": "[UNUSED_TOKEN_39]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92437": { |
| "content": "[UNUSED_TOKEN_40]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92438": { |
| "content": "[UNUSED_TOKEN_41]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92439": { |
| "content": "[UNUSED_TOKEN_42]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92440": { |
| "content": "[UNUSED_TOKEN_43]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92441": { |
| "content": "[UNUSED_TOKEN_44]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92442": { |
| "content": "[UNUSED_TOKEN_45]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92443": { |
| "content": "[UNUSED_TOKEN_46]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92444": { |
| "content": "[UNUSED_TOKEN_47]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92445": { |
| "content": "[UNUSED_TOKEN_48]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92446": { |
| "content": "[UNUSED_TOKEN_49]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92447": { |
| "content": "[UNUSED_TOKEN_50]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92448": { |
| "content": "[UNUSED_TOKEN_51]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92449": { |
| "content": "[UNUSED_TOKEN_52]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92450": { |
| "content": "[UNUSED_TOKEN_53]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92451": { |
| "content": "[UNUSED_TOKEN_54]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92452": { |
| "content": "[UNUSED_TOKEN_55]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92453": { |
| "content": "[UNUSED_TOKEN_56]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92454": { |
| "content": "[UNUSED_TOKEN_57]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92455": { |
| "content": "[UNUSED_TOKEN_58]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92456": { |
| "content": "[UNUSED_TOKEN_59]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92457": { |
| "content": "[UNUSED_TOKEN_60]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92458": { |
| "content": "[UNUSED_TOKEN_61]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92459": { |
| "content": "[UNUSED_TOKEN_62]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92460": { |
| "content": "[UNUSED_TOKEN_63]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92461": { |
| "content": "[UNUSED_TOKEN_64]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92462": { |
| "content": "[UNUSED_TOKEN_65]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92463": { |
| "content": "[UNUSED_TOKEN_66]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92464": { |
| "content": "[UNUSED_TOKEN_67]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92465": { |
| "content": "[UNUSED_TOKEN_68]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92466": { |
| "content": "[UNUSED_TOKEN_69]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92467": { |
| "content": "[UNUSED_TOKEN_70]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92468": { |
| "content": "[UNUSED_TOKEN_71]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92469": { |
| "content": "[UNUSED_TOKEN_72]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92470": { |
| "content": "[UNUSED_TOKEN_73]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92471": { |
| "content": "[UNUSED_TOKEN_74]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92472": { |
| "content": "[UNUSED_TOKEN_75]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92473": { |
| "content": "[UNUSED_TOKEN_76]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92474": { |
| "content": "[UNUSED_TOKEN_77]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92475": { |
| "content": "[UNUSED_TOKEN_78]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92476": { |
| "content": "[UNUSED_TOKEN_79]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92477": { |
| "content": "[UNUSED_TOKEN_80]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92478": { |
| "content": "[UNUSED_TOKEN_81]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92479": { |
| "content": "[UNUSED_TOKEN_82]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92480": { |
| "content": "[UNUSED_TOKEN_83]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92481": { |
| "content": "[UNUSED_TOKEN_84]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92482": { |
| "content": "[UNUSED_TOKEN_85]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92483": { |
| "content": "[UNUSED_TOKEN_86]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92484": { |
| "content": "[UNUSED_TOKEN_87]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92485": { |
| "content": "[UNUSED_TOKEN_88]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92486": { |
| "content": "[UNUSED_TOKEN_89]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92487": { |
| "content": "[UNUSED_TOKEN_90]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92488": { |
| "content": "[UNUSED_TOKEN_91]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92489": { |
| "content": "[UNUSED_TOKEN_92]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92490": { |
| "content": "[UNUSED_TOKEN_93]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92491": { |
| "content": "[UNUSED_TOKEN_94]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92492": { |
| "content": "[UNUSED_TOKEN_95]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92493": { |
| "content": "[UNUSED_TOKEN_96]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92494": { |
| "content": "[UNUSED_TOKEN_97]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92495": { |
| "content": "[UNUSED_TOKEN_98]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92496": { |
| "content": "[UNUSED_TOKEN_99]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92497": { |
| "content": "[UNUSED_TOKEN_100]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92498": { |
| "content": "[UNUSED_TOKEN_101]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92499": { |
| "content": "[UNUSED_TOKEN_102]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92500": { |
| "content": "[UNUSED_TOKEN_103]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92501": { |
| "content": "[UNUSED_TOKEN_104]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92502": { |
| "content": "[UNUSED_TOKEN_105]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92503": { |
| "content": "[UNUSED_TOKEN_106]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92504": { |
| "content": "[UNUSED_TOKEN_107]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92505": { |
| "content": "[UNUSED_TOKEN_108]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92506": { |
| "content": "[UNUSED_TOKEN_109]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92507": { |
| "content": "[UNUSED_TOKEN_110]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92508": { |
| "content": "[UNUSED_TOKEN_111]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92509": { |
| "content": "[UNUSED_TOKEN_112]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92510": { |
| "content": "[UNUSED_TOKEN_113]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92511": { |
| "content": "[UNUSED_TOKEN_114]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92512": { |
| "content": "[UNUSED_TOKEN_115]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92513": { |
| "content": "[UNUSED_TOKEN_116]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92514": { |
| "content": "[UNUSED_TOKEN_117]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92515": { |
| "content": "[UNUSED_TOKEN_118]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92516": { |
| "content": "[UNUSED_TOKEN_119]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92517": { |
| "content": "[UNUSED_TOKEN_120]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92518": { |
| "content": "[UNUSED_TOKEN_121]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92519": { |
| "content": "[UNUSED_TOKEN_122]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92520": { |
| "content": "[UNUSED_TOKEN_123]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92521": { |
| "content": "[UNUSED_TOKEN_124]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92522": { |
| "content": "[UNUSED_TOKEN_125]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92523": { |
| "content": "[UNUSED_TOKEN_126]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92524": { |
| "content": "[UNUSED_TOKEN_127]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92525": { |
| "content": "[UNUSED_TOKEN_128]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92526": { |
| "content": "[UNUSED_TOKEN_129]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92527": { |
| "content": "[UNUSED_TOKEN_130]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92528": { |
| "content": "[UNUSED_TOKEN_131]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92529": { |
| "content": "[UNUSED_TOKEN_132]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92530": { |
| "content": "[UNUSED_TOKEN_133]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92531": { |
| "content": "[UNUSED_TOKEN_134]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92532": { |
| "content": "[UNUSED_TOKEN_135]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92533": { |
| "content": "[UNUSED_TOKEN_136]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92534": { |
| "content": "[UNUSED_TOKEN_137]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92535": { |
| "content": "[UNUSED_TOKEN_138]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92536": { |
| "content": "[UNUSED_TOKEN_139]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92537": { |
| "content": "[UNUSED_TOKEN_140]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92538": { |
| "content": "<|plugin|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92539": { |
| "content": "<|interpreter|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92540": { |
| "content": "<|action_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92541": { |
| "content": "<|action_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92542": { |
| "content": "<|im_end|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92543": { |
| "content": "<|im_start|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": true |
| }, |
| "92544": { |
| "content": "[UNUSED_TOKEN_141]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92545": { |
| "content": "[UNUSED_TOKEN_142]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92546": { |
| "content": "[UNUSED_TOKEN_143]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92547": { |
| "content": "[UNUSED_TOKEN_144]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92548": { |
| "content": "[UNUSED_TOKEN_145]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| }, |
| "92549": { |
| "content": "[UNUSED_TOKEN_146]", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false, |
| "special": false |
| } |
| }, |
| "additional_special_tokens": [ |
| "<|im_start|>", |
| "<|im_end|>", |
| "<|action_start|>", |
| "<|action_end|>", |
| "<|interpreter|>", |
| "<|plugin|>" |
| ], |
| "auto_map": { |
| "AutoTokenizer": [ |
| "tokenization_internlm2.InternLM2Tokenizer", |
| "tokenization_internlm2_fast.InternLM2TokenizerFast" |
| ] |
| }, |
| "bos_token": "<s>", |
| "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", |
| "clean_up_tokenization_spaces": false, |
| "decode_with_prefix_space": false, |
| "eos_token": "</s>", |
| "max_length": 869, |
| "model_max_length": 1000000000000000019884624838656, |
| "pad_token": "</s>", |
| "sp_model_kwargs": null, |
| "stride": 0, |
| "tokenizer_class": "InternLM2Tokenizer", |
| "truncation_side": "right", |
| "truncation_strategy": "longest_first", |
| "unk_token": "<unk>" |
| } |
|
|
