BusyBeaver-50M / tokenizer_config.json
DJLougen's picture
Upload folder using huggingface_hub
32f947d verified
{
"bos_token": "<s>",
"eos_token": "</s>",
"unk_token": "<unk>",
"pad_token": "<pad>",
"additional_special_tokens": [
"<busybeaver_task>",
"</busybeaver_task>",
"<tool_schema>",
"</tool_schema>",
"<|system|>",
"<|goal|>",
"<|state|>",
"<|tools|>",
"<|output_schema|>",
"<|assistant|>"
],
"model_max_length": 2048,
"clean_up_tokenization_spaces": false
}