Upload 3 files

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<bos>",
+      "content": "<｜begin▁of▁sentence｜>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "<eos>",
+      "content": "<｜end▁of▁sentence｜>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "<pad>",
+      "content": "<｜▁pad▁｜>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 99900,
-      "content": "<|user|>",
+      "content": "<｜im▁start｜>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 99901,
-      "content": "<|system|>",
+      "content": "<｜im▁end｜>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -1006,9 +1006,9 @@
     "byte_fallback": false,
     "ignore_merges": false,
     "vocab": {
-      "<bos>": 0,
-      "<eos>": 1,
-      "<pad>": 2,
+      "<｜begin▁of▁sentence｜>": 0,
+      "<｜end▁of▁sentence｜>": 1,
+      "<｜▁pad▁｜>": 2,
       "!": 3,
       "\"": 4,
       "#": 5,

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+    "special_tokens": {
+        "bos": "<｜begin▁of▁sentence｜>",
+        "eos": "<｜end▁of▁sentence｜>",
+        "pad": "<｜▁pad▁｜>",
+        "im_start": "<｜im▁start｜>",
+        "im_end":   "<｜im▁end｜>"
+    },
+    "chat_template":  "{% for message in messages %}{% if message['role'] == 'system' %}<｜im▁start｜>system\n{{ message['content'] }}<｜im▁end｜>\n{% elif message['role'] == 'user' %}<｜im▁start｜>user\n{{ message['content'] }}<｜im▁end｜>\n{% elif message['role'] == 'assistant' %}<｜im▁start｜>assistant\n{{ message['content'] }}<｜im▁end｜>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<｜im▁start｜>assistant\n{% endif %}"
+    
+}