Hugging Face's logo

reaperdoesntknow
/
SharperSwarm

Text Generation
Transformers
Safetensors
English
swarm_agi
swarm
ai
agent
llm
convergent
cpu
fp32
agi
convergentintel
Model card Files
xet
 Community
SharperSwarm / tokenizer_config.json
reaperdoesntknow's picture
reaperdoesntknow
Upload tokenizer
a9f26bf verified
raw
history blame contribute delete
6.94 kB
{
 "add_prefix_space": false,
 "added_tokens_decoder": {
 "50256": {
 "content": "<|endoftext|>",
 "lstrip": false,
 "normalized": true,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50257": {
 "content": "<|pad|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50258": {
 "content": "<|bos|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50259": {
 "content": "<|eos|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50260": {
 "content": "<|unk|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50261": {
 "content": "<|thought|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50262": {
 "content": "<|action|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50263": {
 "content": "<|observe|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50264": {
 "content": "<|route|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50265": {
 "content": "<|meta|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50266": {
 "content": "<|system|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50267": {
 "content": "<|user|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50268": {
 "content": "<|assistant|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50269": {
 "content": "<|tool|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50270": {
 "content": "<|agent:0|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50271": {
 "content": "<|agent:1|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50272": {
 "content": "<|agent:2|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50273": {
 "content": "<|agent:3|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50274": {
 "content": "<|agent:4|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50275": {
 "content": "<|agent:5|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50276": {
 "content": "<|agent:6|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50277": {
 "content": "<|agent:7|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50278": {
 "content": "<|agent:8|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50279": {
 "content": "<|agent:9|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50280": {
 "content": "<|agent:10|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50281": {
 "content": "<|agent:11|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50282": {
 "content": "<|agent:12|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50283": {
 "content": "<|agent:13|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50284": {
 "content": "<|agent:14|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50285": {
 "content": "<|agent:15|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50286": {
 "content": "<|agent:16|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50287": {
 "content": "<|agent:17|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50288": {
 "content": "<|agent:18|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 },
 "50289": {
 "content": "<|agent:19|>",
 "lstrip": false,
 "normalized": false,
 "rstrip": false,
 "single_word": false,
 "special": true
 }
 },
 "additional_special_tokens": [
 "<|agent:0|>",
 "<|agent:1|>",
 "<|agent:2|>",
 "<|agent:3|>",
 "<|agent:4|>",
 "<|agent:5|>",
 "<|agent:6|>",
 "<|agent:7|>",
 "<|agent:8|>",
 "<|agent:9|>",
 "<|agent:10|>",
 "<|agent:11|>",
 "<|agent:12|>",
 "<|agent:13|>",
 "<|agent:14|>",
 "<|agent:15|>",
 "<|agent:16|>",
 "<|agent:17|>",
 "<|agent:18|>",
 "<|agent:19|>"
 ],
 "bos_token": "<|endoftext|>",
 "clean_up_tokenization_spaces": false,
 "eos_token": "<|endoftext|>",
 "extra_special_tokens": {},
 "max_length": 2048,
 "model_max_length": 1024,
 "pad_to_multiple_of": null,
 "pad_token": "<|pad|>",
 "pad_token_type_id": 0,
 "padding_side": "right",
 "stride": 0,
 "tokenizer_class": "GPT2Tokenizer",
 "truncation_side": "right",
 "truncation_strategy": "longest_first",
 "unk_token": "<|endoftext|>"
}