File size: 222 Bytes
2b9fdf9
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
{
  "kind": "byt5",
  "class_name": "ByT5Tokenizer",
  "offset": 1976,
  "pad_id": 1976,
  "num_tokens": 384,
  "aggregate_offset": 1976,
  "byte_offset": 3,
  "eos_token_id": 1,
  "global_eos_token_id": 2361
}