victor HF Staff commited on
Commit
bdf0876
·
verified ·
1 Parent(s): 83237c6

Fix tokenizer_class from TokenizersBackend to PreTrainedTokenizerFast for vLLM compat

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +12 -12
tokenizer_config.json CHANGED
@@ -1,14 +1,14 @@
1
  {
2
- "backend": "tokenizers",
3
- "bos_token": "<|startoftext|>",
4
- "clean_up_tokenization_spaces": false,
5
- "eos_token": "<|return|>",
6
- "is_local": false,
7
- "model_input_names": [
8
- "input_ids",
9
- "attention_mask"
10
- ],
11
- "model_max_length": 1000000000000000019884624838656,
12
- "pad_token": "<|endoftext|>",
13
- "tokenizer_class": "TokenizersBackend"
14
  }
 
1
  {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|return|>",
6
+ "is_local": false,
7
+ "model_input_names": [
8
+ "input_ids",
9
+ "attention_mask"
10
+ ],
11
+ "model_max_length": 131072,
12
+ "pad_token": "<|endoftext|>",
13
+ "tokenizer_class": "PreTrainedTokenizerFast"
14
  }