Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta.json

@@ -0,0 +1,4 @@
+{
+  "torch": "2.11.0",
+  "transformers": "5.5.0"
+}

README.md

@@ -0,0 +1,165 @@
+---
+library_name: transformers
+base_model:
+- MiniMaxAI/MiniMax-M2.7
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [MiniMaxAI/MiniMax-M2.7](https://huggingface.co/MiniMaxAI/MiniMax-M2.7).
+
+| File path | Size |
+|------|------|
+| model.safetensors | 7.4MB |
+
+
+### Example usage:
+
+- vLLM
+
+```bash
+vllm serve tiny-random/minimax-m2.7 \
+    --trust-remote-code \
+    --tensor-parallel-size 1 \
+    --reasoning-parser minimax_m2_append_think \
+    --enable-auto-tool-choice \
+    --tool-call-parser minimax_m2
+```
+
+- SGLang
+
+```bash
+python -m sglang.launch_server \
+    -trust-remote-code \
+    --model-path tiny-random/minimax-m2.7 \
+    --tp-size 1 \
+    --tool-call-parser minimax-m2 \
+    --reasoning-parser minimax-append-think
+```
+
+- Transformers
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+model_id = "tiny-random/minimax-m2.7"
+tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    dtype=torch.bfloat16,
+    trust_remote_code=True,
+)
+pipe = pipeline('text-generation', model=model,
+                tokenizer=tokenizer, trust_remote_code=True)
+print(pipe('Write an article about Artificial Intelligence.', max_new_tokens=16))
+```
+
+### Codes to create this repo:
+
+<details>
+<summary>Click to expand</summary>
+
+```python
+import json
+from pathlib import Path
+
+import accelerate
+import torch
+import transformers
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "MiniMaxAI/MiniMax-M2.7"
+save_folder = "/tmp/tiny-random/minimax-m27"
+
+processor = AutoTokenizer.from_pretrained(source_model_id)
+processor.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json = json.load(f)
+
+del config_json['auto_map'] # is already supported in transformers codebase
+config_json["attn_type_list"] = [1, 1]
+config_json['head_dim'] = 32
+config_json['hidden_size'] = 8
+config_json['intermediate_size'] = 32
+config_json['num_attention_heads'] = 8
+config_json['num_key_value_heads'] = 4
+config_json['num_hidden_layers'] = 2
+config_json['mlp_intermediate_size'] = 32
+config_json['rotary_dim'] = 16
+del config_json['quantization_config']
+
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModelForCausalLM.from_config(config, trust_remote_code=True)
+torch.set_default_dtype(torch.float32)
+print(model)
+
+# according to source model, gate is in FP32
+for i in range(config.num_hidden_layers):
+    model.model.layers[i].mlp.gate = model.model.layers[i].mlp.gate.float()
+    model.model.layers[i].mlp.e_score_correction_bias = model.model.layers[i].mlp.e_score_correction_bias.float()
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+set_seed(42)
+model = model.cpu()
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.1)
+        print(name, p.shape)
+model.save_pretrained(save_folder)
+print(model)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+MiniMaxM2ForCausalLM(
+  (model): MiniMaxM2Model(
+    (embed_tokens): Embedding(200064, 8)
+    (layers): ModuleList(
+      (0-1): 2 x MiniMaxM2DecoderLayer(
+        (self_attn): MiniMaxM2Attention(
+          (q_proj): Linear(in_features=8, out_features=256, bias=False)
+          (k_proj): Linear(in_features=8, out_features=128, bias=False)
+          (v_proj): Linear(in_features=8, out_features=128, bias=False)
+          (o_proj): Linear(in_features=256, out_features=8, bias=False)
+          (q_norm): MiniMaxM2RMSNorm((256,), eps=1e-06)
+          (k_norm): MiniMaxM2RMSNorm((128,), eps=1e-06)
+        )
+        (mlp): MiniMaxM2SparseMoeBlock(
+          (gate): MiniMaxM2TopKRouter()
+          (experts): MiniMaxM2Experts(
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): MiniMaxM2RMSNorm((8,), eps=1e-06)
+        (post_attention_layernorm): MiniMaxM2RMSNorm((8,), eps=1e-06)
+      )
+    )
+    (norm): MiniMaxM2RMSNorm((8,), eps=1e-06)
+    (rotary_emb): MiniMaxM2RotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=8, out_features=200064, bias=False)
+)
+```
+
+</details>

chat_template.jinja

@@ -0,0 +1,159 @@
+{# ----------‑‑‑ special token variables ‑‑‑---------- #}
+{%- set toolcall_begin_token   = '<minimax:tool_call>'         -%}
+{%- set toolcall_end_token     = '</minimax:tool_call>'        -%}
+{#- Tool Rendering Functions ============================================== -#}
+{%- macro render_tool_namespace(namespace_name, tool_list) -%}
+{%- for tool in tool_list -%}
+<tool>{{ tool.function | tojson(ensure_ascii=False) }}</tool>
+{% endfor -%}
+{%- endmacro -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{ content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{#- System Message Construction ============================================ -#}
+{%- macro build_system_message(system_message) -%}
+    {%- if system_message and system_message.content -%}
+        {{- visible_text(system_message.content) }}
+    {%- else -%}
+        {%- if model_identity is not defined -%}
+            {%- set model_identity = "You are a helpful assistant. Your name is MiniMax-M2.7 and is built by MiniMax." -%}
+        {%- endif -%}
+        {{- model_identity }}
+    {%- endif -%}
+    
+    {#- Handle current_date -#}
+    {%- if system_message and system_message.current_date -%}
+        {{- '\n' ~ 'Current date: ' + system_message.current_date }}
+    {%- endif -%}
+    {#- Handle current_location -#}
+    {%- if system_message and system_message.current_location -%}
+        {{- '\n' ~ 'Current location: ' + system_message.current_location }}
+    {%- endif -%}
+{%- endmacro -%}
+{#- Main Template Logic ================================================= -#}
+{#- Extract system message (only first message if it's system) -#}
+{%- set system_message = none -%}
+{%- set conversation_messages = messages -%}
+{%- if messages and messages[0].role == "system" -%}
+    {%- set system_message = messages[0] -%}
+    {%- set conversation_messages = messages[1:] -%}
+{%- endif -%}
+{#- Get the last user message turn, for interleved thinking -#}
+{%- set ns = namespace(last_user_index=-1) %}
+{% for m in conversation_messages %}
+    {%- if m.role == 'user' %}
+        {% set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{#- Render system message -#}
+{{- ']~!b[' ~ ']~b]system' ~ '\n' }}
+{{- build_system_message(system_message) }}
+{#- Render tools if available -#}
+{%- if tools -%}
+    {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }}
+    {{- '\n' ~ '<tools>' ~ '\n' }}
+    {{- render_tool_namespace("functions", tools) }}
+    {{- '</tools>' ~ '\n\n' }}
+{{- 'When making tool calls, use XML format to invoke tools and pass parameters:' ~ '\n' }}
+{{- '\n' ~ toolcall_begin_token }}
+<invoke name="tool-name-1">
+<parameter name="param-key-1">param-value-1</parameter>
+<parameter name="param-key-2">param-value-2</parameter>
+...
+</invoke>
+{{- '\n' ~ toolcall_end_token }}
+{%- endif -%}
+{{- '[e~[\n' }}
+
+{#- Render messages -#}
+{%- set last_tool_call = namespace(name=none) -%}
+{%- for message in conversation_messages -%}
+    {%- if message.role == 'assistant' -%}
+        {#- Only render reasoning_content if no user message follows -#}
+        {{- ']~b]ai' ~ '\n' }}
+
+        {%- set reasoning_content = '' %}
+        {%- set content = visible_text(message.content) %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].strip('\n').split('<think>')[-1].strip('\n') %}
+                {%- set content = content.split('</think>')[-1].strip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if reasoning_content and loop.index0 > ns.last_user_index -%}
+            {{- '<think>' ~ '\n' ~ reasoning_content ~ '\n' ~ '</think>' ~ '\n\n' }}
+        {%- endif -%}
+        {%- if content -%}
+            {{- content }}
+        {%- endif -%}
+        {%- if message.tool_calls -%}
+            {{- '\n' ~ toolcall_begin_token ~ '\n' }}
+
+            {%- for tool_call in message.tool_calls -%}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<invoke name="' + tool_call.name + '">' }}
+                {% set _args = tool_call.arguments %}
+                {%- for k, v in _args.items() %}
+                {{- '<parameter name="' + k + '">' }}
+                {{- v | tojson(ensure_ascii=False) if v is not string else v }}
+                {{- '</parameter>' }}
+                {% endfor %}
+                {{- '</invoke>' ~ '\n' }}
+            {%- endfor -%}
+            
+            {{- toolcall_end_token}}
+            {%- set last_tool_call.name = message.tool_calls[-1].name -%}
+        {%- else -%}
+            {%- set last_tool_call.name = none -%}
+        {%- endif -%}
+        {{- '[e~[' ~ '\n' }}
+        
+    {%- elif message.role == 'tool' -%}
+    {%- if last_tool_call.name is none -%}
+        {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
+    {%- endif -%}
+    {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%}
+        {{- ']~b]tool' }}
+    {%- endif -%}
+    {%- if message.content is string -%}
+        {{- '\n<response>' }}
+        {{- message.content }}
+        {{- '</response>' }}
+    {%- else -%}
+        {%- for tr in message.content -%}
+            {{- '\n<response>' }}
+            {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }}
+            {{- '\n</response>' }}
+        {%- endfor -%}
+    {%- endif -%}
+    {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%}
+        {{- '[e~[\n' -}}
+    {%- endif -%}
+        
+    {%- elif message.role == 'user' -%}
+        {{- ']~b]user' ~ '\n' }}
+        {{- visible_text(message.content) }}
+        {{- '[e~[' ~ '\n' }}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Generation prompt -#}
+{%- if add_generation_prompt -%}
+{{- ']~b]ai' ~ '\n' ~ '<think>' ~ '\n' }}
+{%- endif -%}

config.json

@@ -0,0 +1,48 @@
+{
+  "architectures": [
+    "MiniMaxM2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_type_list": [
+    1,
+    1
+  ],
+  "bos_token_id": 200034,
+  "dtype": "bfloat16",
+  "eos_token_id": 200020,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 8,
+  "initializer_range": 0.02,
+  "intermediate_size": 32,
+  "max_position_embeddings": 196608,
+  "mlp_intermediate_size": 32,
+  "model_type": "minimax_m2",
+  "mtp_transformer_layers": 1,
+  "num_attention_heads": 8,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "num_local_experts": 256,
+  "num_mtp_modules": 3,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "qk_norm_type": "per_layer",
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 5000000,
+    "rope_type": "default"
+  },
+  "rotary_dim": 16,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "scoring_func": "sigmoid",
+  "shared_intermediate_size": 0,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "use_mtp": true,
+  "use_qk_norm": true,
+  "use_routing_bias": true,
+  "vocab_size": 200064
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 200019,
+  "do_sample": true,
+  "eos_token_id": 200020,
+  "temperature": 1.0,
+  "top_k": 40,
+  "top_p": 0.95,
+  "transformers_version": "5.5.0",
+  "trust_remote_code": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f279f05fbc6f7d57725c55179104dd39c644b1c16db6e020617c5cf0755537b
+size 7420752

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b81e5e5cba2b169e86a0771825a927e9d41b4c4484ded4a286410f41f702f17
+size 15523144

tokenizer_config.json

@@ -0,0 +1,63 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "]~!b[",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "[e~[",
+  "extra_special_tokens": [
+    "<code_interpreter>",
+    "<commit_after>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<empty_output>",
+    "<filename>",
+    "<fim_middle>",
+    "<fim_pad>",
+    "<fim_prefix>",
+    "<fim_suffix>",
+    "<function_call>",
+    "<gh_stars>",
+    "]<]speech[>[",
+    "]<]image[>[",
+    "]<]video[>[",
+    "]<]start of speech[>[",
+    "]<]end of speech[>[",
+    "]<]start of image[>[",
+    "]<]end of image[>[",
+    "]<]start of video[>[",
+    "]<]end of video[>[",
+    "]<]vision pad[>[",
+    "]~!b[",
+    "<issue_closed>",
+    "<issue_comment>",
+    "<issue_start>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<reponame>",
+    "[e~[",
+    "]!d~[",
+    "]!p~[",
+    "]~b]",
+    "<jupyter_error>",
+    "<add_file>",
+    "<delete_file>",
+    "<rename_file>",
+    "<edit_file>",
+    "<commit_message>",
+    "<empty_source_file>",
+    "<repo_struct>",
+    "<code_context>",
+    "<file_content>",
+    "<source_files>",
+    "<pr_start>",
+    "<review_comment>",
+    "<filepath>",
+    "<file_sep>"
+  ],
+  "is_local": false,
+  "model_max_length": 40960000,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "]!d~["
+}