Upload tokenizer for dual-GPU DeepSpeed full finetune

chat_template.jinja

@@ -0,0 +1,139 @@
+{%- if not add_generation_prompt is defined %}
+    {%- set add_generation_prompt = false %}
+{%- endif %}
+{%- set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_first_user=true, is_last_user=false) %}
+{%- for message in messages %}
+    {%- if message['role'] == 'system' %}
+        {%- if ns.is_first_sp %}
+            {%- set ns.system_prompt = ns.system_prompt + message['content'] %}
+            {%- set ns.is_first_sp = false %}
+        {%- else %}
+            {% set ns.system_prompt = ns.system_prompt + '
+
+' + message['content'] %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{{- bos_token }}
+{{- ns.system_prompt }}
+{%- if tools %}
+    {%- if ns.system_prompt != '' %}
+        {{- '
+
+# Tools
+
+You may call one or more functions to assist with the user query.' }}
+    {%- else %}
+        {{- '# Tools
+
+You may call one or more functions to assist with the user query.' }}
+    {%- endif %}
+    {{- '
+
+You are provided with function signatures within <tools></tools> XML tags:' }}
+    {{- '
+<tools>
+' }}
+    {%- for tool in tools %}
+        {%- if loop.index0 > 1 %}
+            {{- '
+' }}
+        {%- endif %}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- '
+</tools>
+
+' }}
+    {{- 'For function call returns, you should first print <tool_calls>' }}
+    {{- 'For each function call, you should return object like:
+' }}
+    {{- '<tool_call>function_name
+```json
+function_arguments_in_json_format
+```</tool_call>' }}
+    {{- 'At the end of function call returns, you should print </tool_calls>' }}
+{%- endif %}
+{%- if ns.system_prompt != '' or tools %}
+    {{- '<｜hy_place▁holder▁no▁3｜>' }}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message['role'] == 'user' %}
+        {%- set ns.is_tool = false %}
+        {%- set ns.is_first = false %}
+        {%- set ns.is_last_user = true %}
+        {{- '<｜hy_User｜>' + message['content'] + '<｜hy_Assistant｜>' }}
+    {%- endif %}
+    {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}
+        {%- set ns.is_last_user = false %}
+        {%- if ns.is_tool %}
+            {{- '</tool_responses>' + '<｜hy_Assistant｜>' }}
+        {%- endif %}
+        {%- set ns.is_first = false %}
+        {%- set ns.is_tool = false %}
+        {%- set ns.is_output_first = true %}
+        {%- for tool in message['tool_calls'] %}
+            {%- set arguments = tool['function']['arguments'] %}
+            {%- if arguments is not string %}
+                {%- set arguments = arguments | tojson %}
+            {%- endif %}
+            {%- if not ns.is_first %}
+                {%- if message['content'] is none %}
+                    {{- '<tool_calls><tool_call>' + tool['function']['name'] + '
+' + '```json' + '
+' + arguments + '
+' + '```' + '</tool_call>' }}
+                {%- else %}
+                    {{- message['content'] + '<tool_calls><tool_call>' + tool['function']['name'] + '
+' + '```json' + '
+' + arguments + '
+' + '```' + '</tool_call>' }}
+                {%- endif %}
+            {%- set ns.is_first = true %}
+            {%- else %}
+                {{- '
+' + '<tool_call>' + tool['function']['name'] + '
+' + '```json' + '
+' + arguments + '
+' + '```' + '</tool_call>' }}
+            {%- endif %}
+        {%- endfor %}
+        {{- '</tool_calls>' + eos_token }}
+    {%- endif %}
+    {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %}
+        {%- set content = message['content'] %}
+        {%- if '<answer>' in content and not loop.last %}
+            {%- set content = content.split('<answer>')[-1].strip('</answer>').strip() %}
+        {%- endif %}
+        {%- set ns.is_last_user = false %}
+        {%- if ns.is_tool %}
+            {{- '</tool_responses>' + '<｜hy_Assistant｜>' + content + eos_token }}
+            {%- set ns.is_tool = false %}
+        {%- else %}
+            {{- content + eos_token }}
+        {%- endif %}
+    {%- endif %}
+    {%- if message['role'] == 'tool' %}
+        {%- set ns.is_last_user = false %}
+        {%- set ns.is_tool = true %}
+        {%- if ns.is_output_first %}
+            {{- '<｜hy_User｜>' + '<tool_responses><tool_response>' + message['content'] + '</tool_response>' }}
+            {%- set ns.is_output_first = false %}
+        {%- else %}
+            {{- '
+<tool_response>' + message['content'] + '</tool_response>' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.is_tool %}
+    {{- '</tool_responses>' + '<｜hy_Assistant｜>' }}
+{%- endif %}
+{%- if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}
+    {{- '<｜hy_Assistant｜>' }}
+{%- endif %}
+{%- if enable_thinking is defined and not enable_thinking %}
+    {{- '<think>
+
+</think>
+' }}
+{%- endif %}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜hy_begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜hy_place▁holder▁no▁2｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜hy_▁pad▁｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}