Hugging Face's logo

AksaraLLM
/
aksara-tokenizer-v3

Model card Files
xet
 Community
aksara-tokenizer-v3
File size: 642 Bytes
faf515e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
{
  "version": "3.0.0",
  "vocab_size": 32773,
  "pre_tokenizer": "Whitespace",
  "normalizer": "NFKC (NO lowercase)",
  "model": "BPE",
  "brand_tokens": [
    "AksaraLLM",
    "aksarallm",
    "AKSARALLM",
    "Indonesia",
    "indonesia",
    "INDONESIA",
    "Pancasila",
    "pancasila",
    "Nusantara",
    "nusantara"
  ],
  "special_tokens": {
    "[PAD]": 0,
    "[EOS]": 1,
    "[BOS]": 2,
    "[UNK]": 3,
    "[SEP]": 4,
    "[MASK]": 5,
    "[INST]": 6,
    "[/INST]": 7,
    "[SYS]": 8,
    "[USER]": 9,
    "[ASST]": 10,
    "[TURN]": 11,
    "[LANG_ID]": 12,
    "[LANG_JV]": 13,
    "[LANG_SU]": 14,
    "[LANG_EN]": 15
  }
}