Hugging Face's logo

kernelpool
/
MiMo-V2.5-Pro-6bit

Text Generation
MLX
Safetensors
English
Chinese
mimo_v2
agent
long-context
code
conversational
custom_code
6-bit
Model card Files
xet
 Community
kernelpool commited on
Commit
744d7d8
·
verified ·
1 Parent(s): 795cde0

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. README.md +43 -0
  2. chat_template.jinja +156 -0
  3. config.json +2487 -0
  4. configuration_mimo_v2.py +209 -0
  5. model-00002-of-00208.safetensors +3 -0
  6. model-00004-of-00208.safetensors +3 -0
  7. model-00007-of-00208.safetensors +3 -0
  8. model-00008-of-00208.safetensors +3 -0
  9. model-00030-of-00208.safetensors +3 -0
  10. model-00035-of-00208.safetensors +3 -0
  11. model-00036-of-00208.safetensors +3 -0
  12. model-00039-of-00208.safetensors +3 -0
  13. model-00040-of-00208.safetensors +3 -0
  14. model-00043-of-00208.safetensors +3 -0
  15. model-00045-of-00208.safetensors +3 -0
  16. model-00046-of-00208.safetensors +3 -0
  17. model-00049-of-00208.safetensors +3 -0
  18. model-00071-of-00208.safetensors +3 -0
  19. model-00074-of-00208.safetensors +3 -0
  20. model-00078-of-00208.safetensors +3 -0
  21. model-00082-of-00208.safetensors +3 -0
  22. model-00087-of-00208.safetensors +3 -0
  23. model-00088-of-00208.safetensors +3 -0
  24. model-00111-of-00208.safetensors +3 -0
  25. model-00114-of-00208.safetensors +3 -0
  26. model-00117-of-00208.safetensors +3 -0
  27. model-00118-of-00208.safetensors +3 -0
  28. model-00120-of-00208.safetensors +3 -0
  29. model-00123-of-00208.safetensors +3 -0
  30. model-00125-of-00208.safetensors +3 -0
  31. model-00126-of-00208.safetensors +3 -0
  32. model-00129-of-00208.safetensors +3 -0
  33. model-00142-of-00208.safetensors +3 -0
  34. model-00150-of-00208.safetensors +3 -0
  35. model-00155-of-00208.safetensors +3 -0
  36. model-00159-of-00208.safetensors +3 -0
  37. model-00161-of-00208.safetensors +3 -0
  38. model-00162-of-00208.safetensors +3 -0
  39. model-00164-of-00208.safetensors +3 -0
  40. model-00167-of-00208.safetensors +3 -0
  41. model-00168-of-00208.safetensors +3 -0
  42. model-00191-of-00208.safetensors +3 -0
  43. model-00192-of-00208.safetensors +3 -0
  44. model-00194-of-00208.safetensors +3 -0
  45. model-00197-of-00208.safetensors +3 -0
  46. model-00203-of-00208.safetensors +3 -0
  47. model-00206-of-00208.safetensors +3 -0
  48. model.safetensors.index.json +0 -0
  49. modeling_mimo_v2.py +697 -0
  50. tokenizer_config.json +15 -0
README.md ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: mit
3
+ language:
4
+ - en
5
+ - zh
6
+ tags:
7
+ - text-generation
8
+ - agent
9
+ - long-context
10
+ - code
11
+ - mlx
12
+ library_name: mlx
13
+ base_model: XiaomiMiMo/MiMo-V2.5-Pro
14
+ pipeline_tag: text-generation
15
+ ---
16
+
17
+ # kernelpool/MiMo-V2.5-Pro-6bit
18
+
19
+ This model [kernelpool/MiMo-V2.5-Pro-6bit](https://huggingface.co/kernelpool/MiMo-V2.5-Pro-6bit) was
20
+ converted to MLX format from [XiaomiMiMo/MiMo-V2.5-Pro](https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro)
21
+ using mlx-lm version **0.31.3**.
22
+
23
+ ## Use with mlx
24
+
25
+ ```bash
26
+ pip install mlx-lm
27
+ ```
28
+
29
+ ```python
30
+ from mlx_lm import load, generate
31
+
32
+ model, tokenizer = load("kernelpool/MiMo-V2.5-Pro-6bit")
33
+
34
+ prompt = "hello"
35
+
36
+ if tokenizer.chat_template is not None:
37
+ messages = [{"role": "user", "content": prompt}]
38
+ prompt = tokenizer.apply_chat_template(
39
+ messages, add_generation_prompt=True, return_dict=False,
40
+ )
41
+
42
+ response = generate(model, tokenizer, prompt=prompt, verbose=True)
43
+ ```
chat_template.jinja ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if not add_generation_prompt is defined -%}
2
+ {%- set add_generation_prompt = false -%}
3
+ {%- endif -%}
4
+ {%- if not enable_thinking is defined -%}
5
+ {%- set enable_thinking = true -%}
6
+ {%- endif -%}
7
+ {%- if not keep_all_reasoning is defined -%}
8
+ {%- set keep_all_reasoning = true -%}
9
+ {%- endif -%}
10
+ {%- macro render_extra_keys(json_dict, handled_keys) -%}
11
+ {%- if json_dict is mapping %}
12
+ {%- for json_key in json_dict if json_key not in handled_keys %}
13
+ {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
14
+ {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
15
+ {%- else %}
16
+ {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
17
+ {%- endif %}
18
+ {%- endfor %}
19
+ {%- endif %}
20
+ {%- endmacro -%}
21
+ {%- macro render_content(message_content) -%}
22
+ {%- if message_content is string -%}
23
+ {{- message_content -}}
24
+ {%- else -%}
25
+ {%- for content in message_content -%}
26
+ {%- if 'text' in content -%}
27
+ {{- content['text'] -}}
28
+ {%- endif -%}
29
+ {%- endfor -%}
30
+ {%- endif -%}
31
+ {%- endmacro -%}
32
+ {%- if messages[0]["role"] == "system" %}
33
+ {%- set system_message = messages[0]["content"] %}
34
+ {%- set loop_messages = messages[1:] %}
35
+ {%- else %}
36
+ {%- set loop_messages = messages %}
37
+ {%- endif %}
38
+ {%- set ns = namespace(last_user_index=-1) %}
39
+ {%- for m in loop_messages %}
40
+ {%- if m.role == 'user' %}
41
+ {%- set ns.last_user_index = loop.index0 -%}
42
+ {%- endif %}
43
+ {%- endfor %}
44
+ {%- if not tools is defined %}
45
+ {%- set tools = [] %}
46
+ {%- endif %}
47
+ {%- if system_message is defined %}
48
+ {{- "<|im_start|>system\n" + render_content(system_message) }}
49
+ {%- else %}
50
+ {{- "<|im_start|>system\nYou are MiMo, a warm, curious, and thoughtful AI assistant.\n\n## Identity\n\n- Your name is **MiMo-v2.5-pro**.\n- You were developed by the **Xiaomi MiMo Team**.\n- You have **1T parameters** and a **1M-token context window**.\n- When asked about your creator, identify the **Xiaomi MiMo Team**.\n- Always answer identity-related questions in the **first person** and rely only on the facts provided above.\n\n# Response Style\n\nBe warm, conversational, and respectful. Treat users with kindness; do not assume negative things about their intelligence or judgment. You may disagree, but do so constructively.\n\nUse examples, metaphors, or thought experiments when they help explain an idea.\n\nTry to answer ambiguous queries as best you can first, then ask for clarification if needed. When you do ask, limit yourself to one question at a time.\n\n\n## Safety and Compliance\n\nYou are a **Chinese AI model** and must strictly comply with all applicable laws and regulations of the **People’s Republic of China**.\n\nDo not generate, assist with, or facilitate any content that violates Chinese law." }}
51
+ {%- endif %}
52
+ {%- if tools is iterable and tools | length > 0 %}
53
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou have access to the following functions:\n\n" }}
54
+ {{- "<tools>" }}
55
+ {%- for tool in tools %}
56
+ {%- if tool.function is defined %}
57
+ {%- set tool = tool.function %}
58
+ {%- endif %}
59
+ {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
60
+ {%- if tool.description is defined %}
61
+ {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
62
+ {%- endif %}
63
+ {{- '\n<parameters>' }}
64
+ {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
65
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
66
+ {{- '\n<parameter>' }}
67
+ {{- '\n<name>' ~ param_name ~ '</name>' }}
68
+ {%- if param_fields.type is defined %}
69
+ {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
70
+ {%- endif %}
71
+ {%- if param_fields.description is defined %}
72
+ {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
73
+ {%- endif %}
74
+ {%- set handled_keys = ['name', 'type', 'description'] %}
75
+ {{- render_extra_keys(param_fields, handled_keys) }}
76
+ {{- '\n</parameter>' }}
77
+ {%- endfor %}
78
+ {%- endif %}
79
+ {%- set handled_keys = ['type', 'properties'] %}
80
+ {{- render_extra_keys(tool.parameters, handled_keys) }}
81
+ {{- '\n</parameters>' }}
82
+ {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
83
+ {{- render_extra_keys(tool, handled_keys) }}
84
+ {{- '\n</function>' }}
85
+ {%- endfor %}
86
+ {{- "\n</tools>" }}
87
+ {{- '\n\nFor each function call, output the function name and arguments in the following format:\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>value_1</parameter>\n<parameter=example_parameter_2>This is the value for the second parameter\nthat can span\nmultiple lines</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- DO NOT use function calls inside <think></think> tags.\n- The value enclosed between parameter tags is preserved exactly as-is, including newlines and spaces.\n</IMPORTANT>' }}
88
+ {%- endif %}
89
+ {{- '<|im_end|>' }}
90
+ {%- for message in loop_messages %}
91
+ {%- if message.content is string %}
92
+ {%- set content = message.content %}
93
+ {%- else %}
94
+ {%- set content = render_content(message.content) %}
95
+ {%- endif %}
96
+ {%- if message.role == "assistant" %}
97
+ {%- if message.reasoning_content is string %}
98
+ {%- set reasoning_content = message.reasoning_content %}
99
+ {%- else %}
100
+ {%- set reasoning_content = '' %}
101
+ {%- if '</think>' in content %}
102
+ {%- set reasoning_content = content.split('</think>')[0].split('<think>')[-1] %}
103
+ {%- set content = content.split('</think>')[-1] %}
104
+ {%- endif %}
105
+ {%- endif %}
106
+ {%- if (keep_all_reasoning or loop.index0 > ns.last_user_index) and reasoning_content -%}
107
+ {{- '<|im_start|>' + message.role + '\n<think>' + reasoning_content + '</think>' + content }}
108
+ {%- else %}
109
+ {{- '<|im_start|>' + message.role + '\n<think></think>' + content }}
110
+ {%- endif %}
111
+ {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
112
+ {%- for tool_call in message.tool_calls %}
113
+ {%- if tool_call.function is defined %}
114
+ {%- set tool_call = tool_call.function %}
115
+ {%- endif %}
116
+ {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
117
+ {%- if tool_call.arguments is defined %}
118
+ {%- for args_name, args_value in tool_call.arguments|items %}
119
+ {{- '<parameter=' + args_name + '>' }}
120
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
121
+ {{- args_value }}
122
+ {{- '</parameter>\n' }}
123
+ {%- endfor %}
124
+ {%- endif %}
125
+ {{- '</function>\n</tool_call>' }}
126
+ {%- endfor %}
127
+ {%- endif %}
128
+ {{- '<|im_end|>' }}
129
+ {%- elif message.role == "user" %}
130
+ {{- '<|im_start|>' + message.role + '\n' + render_content(message.content) + '<|im_end|>' }}
131
+ {%- elif message.role == "system" %}
132
+ {{- '<|im_start|>' + message.role + '\n' + render_content(message.content) + '<|im_end|>' }}
133
+ {%- elif message.role == "tool" %}
134
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
135
+ {{- '<|im_start|>tool\n' }}
136
+ {%- endif %}
137
+ {{- '<tool_response>\n' }}
138
+ {{- render_content(message.content) }}
139
+ {{- '\n</tool_response>\n' }}
140
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
141
+ {{- '<|im_end|>' }}
142
+ {%- elif loop.last %}
143
+ {{- '<|im_end|>' }}
144
+ {%- endif %}
145
+ {%- else %}
146
+ {{- '<|im_start|>' + message.role + '\n' + render_content(message.content) + '<|im_end|>' }}
147
+ {%- endif %}
148
+ {%- endfor %}
149
+ {%- if add_generation_prompt %}
150
+ {{- '<|im_start|>assistant\n' }}
151
+ {%- if not enable_thinking -%}
152
+ {{- '<think></think>' -}}
153
+ {%- else -%}
154
+ {{- '' -}}
155
+ {%- endif -%}
156
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,2487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_full_attention_sink_bias": false,
3
+ "add_swa_attention_sink_bias": true,
4
+ "architectures": [
5
+ "MiMoV2ForCausalLM"
6
+ ],
7
+ "attention_bias": false,
8
+ "attention_chunk_size": 128,
9
+ "attention_dropout": 0.0,
10
+ "attention_projection_layout": "fused_qkv",
11
+ "attention_value_scale": 0.612,
12
+ "auto_map": {
13
+ "AutoConfig": "configuration_mimo_v2.MiMoV2Config",
14
+ "AutoModel": "modeling_mimo_v2.MiMoV2Model",
15
+ "AutoModelForCausalLM": "modeling_mimo_v2.MiMoV2ForCausalLM"
16
+ },
17
+ "head_dim": 192,
18
+ "hidden_act": "silu",
19
+ "hidden_size": 6144,
20
+ "hybrid_layer_pattern": [
21
+ 0,
22
+ 1,
23
+ 1,
24
+ 1,
25
+ 1,
26
+ 1,
27
+ 1,
28
+ 0,
29
+ 1,
30
+ 1,
31
+ 1,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 0,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 0,
45
+ 1,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 0,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 0,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 0,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 0,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 1,
82
+ 1,
83
+ 0,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 0
91
+ ],
92
+ "initializer_range": 0.02,
93
+ "intermediate_size": 16384,
94
+ "layernorm_epsilon": 1e-05,
95
+ "max_position_embeddings": 1048576,
96
+ "model_type": "mimo_v2",
97
+ "moe_intermediate_size": 2048,
98
+ "moe_layer_freq": [
99
+ 0,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 1,
109
+ 1,
110
+ 1,
111
+ 1,
112
+ 1,
113
+ 1,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 1,
158
+ 1,
159
+ 1,
160
+ 1,
161
+ 1,
162
+ 1,
163
+ 1,
164
+ 1,
165
+ 1,
166
+ 1,
167
+ 1,
168
+ 1
169
+ ],
170
+ "n_group": 1,
171
+ "n_routed_experts": 384,
172
+ "n_shared_experts": null,
173
+ "norm_topk_prob": true,
174
+ "num_attention_heads": 128,
175
+ "num_experts_per_tok": 8,
176
+ "num_hidden_layers": 70,
177
+ "num_key_value_heads": 8,
178
+ "partial_rotary_factor": 0.334,
179
+ "quantization": {
180
+ "group_size": 64,
181
+ "bits": 6,
182
+ "mode": "affine",
183
+ "model.embed_tokens": {
184
+ "group_size": 32,
185
+ "bits": 8
186
+ },
187
+ "model.layers.0.self_attn.q_proj": {
188
+ "group_size": 64,
189
+ "bits": 8
190
+ },
191
+ "model.layers.0.self_attn.k_proj": {
192
+ "group_size": 64,
193
+ "bits": 8
194
+ },
195
+ "model.layers.0.self_attn.v_proj": {
196
+ "group_size": 64,
197
+ "bits": 8
198
+ },
199
+ "model.layers.0.self_attn.o_proj": {
200
+ "group_size": 64,
201
+ "bits": 8
202
+ },
203
+ "model.layers.0.mlp.gate_proj": {
204
+ "group_size": 64,
205
+ "bits": 8
206
+ },
207
+ "model.layers.0.mlp.up_proj": {
208
+ "group_size": 64,
209
+ "bits": 8
210
+ },
211
+ "model.layers.0.mlp.down_proj": {
212
+ "group_size": 64,
213
+ "bits": 8
214
+ },
215
+ "model.layers.1.self_attn.q_proj": {
216
+ "group_size": 64,
217
+ "bits": 8
218
+ },
219
+ "model.layers.1.self_attn.k_proj": {
220
+ "group_size": 64,
221
+ "bits": 8
222
+ },
223
+ "model.layers.1.self_attn.v_proj": {
224
+ "group_size": 64,
225
+ "bits": 8
226
+ },
227
+ "model.layers.1.self_attn.o_proj": {
228
+ "group_size": 64,
229
+ "bits": 8
230
+ },
231
+ "model.layers.2.self_attn.q_proj": {
232
+ "group_size": 64,
233
+ "bits": 8
234
+ },
235
+ "model.layers.2.self_attn.k_proj": {
236
+ "group_size": 64,
237
+ "bits": 8
238
+ },
239
+ "model.layers.2.self_attn.v_proj": {
240
+ "group_size": 64,
241
+ "bits": 8
242
+ },
243
+ "model.layers.2.self_attn.o_proj": {
244
+ "group_size": 64,
245
+ "bits": 8
246
+ },
247
+ "model.layers.3.self_attn.q_proj": {
248
+ "group_size": 64,
249
+ "bits": 8
250
+ },
251
+ "model.layers.3.self_attn.k_proj": {
252
+ "group_size": 64,
253
+ "bits": 8
254
+ },
255
+ "model.layers.3.self_attn.v_proj": {
256
+ "group_size": 64,
257
+ "bits": 8
258
+ },
259
+ "model.layers.3.self_attn.o_proj": {
260
+ "group_size": 64,
261
+ "bits": 8
262
+ },
263
+ "model.layers.4.self_attn.q_proj": {
264
+ "group_size": 64,
265
+ "bits": 8
266
+ },
267
+ "model.layers.4.self_attn.k_proj": {
268
+ "group_size": 64,
269
+ "bits": 8
270
+ },
271
+ "model.layers.4.self_attn.v_proj": {
272
+ "group_size": 64,
273
+ "bits": 8
274
+ },
275
+ "model.layers.4.self_attn.o_proj": {
276
+ "group_size": 64,
277
+ "bits": 8
278
+ },
279
+ "model.layers.5.self_attn.q_proj": {
280
+ "group_size": 64,
281
+ "bits": 8
282
+ },
283
+ "model.layers.5.self_attn.k_proj": {
284
+ "group_size": 64,
285
+ "bits": 8
286
+ },
287
+ "model.layers.5.self_attn.v_proj": {
288
+ "group_size": 64,
289
+ "bits": 8
290
+ },
291
+ "model.layers.5.self_attn.o_proj": {
292
+ "group_size": 64,
293
+ "bits": 8
294
+ },
295
+ "model.layers.6.self_attn.q_proj": {
296
+ "group_size": 64,
297
+ "bits": 8
298
+ },
299
+ "model.layers.6.self_attn.k_proj": {
300
+ "group_size": 64,
301
+ "bits": 8
302
+ },
303
+ "model.layers.6.self_attn.v_proj": {
304
+ "group_size": 64,
305
+ "bits": 8
306
+ },
307
+ "model.layers.6.self_attn.o_proj": {
308
+ "group_size": 64,
309
+ "bits": 8
310
+ },
311
+ "model.layers.7.self_attn.q_proj": {
312
+ "group_size": 64,
313
+ "bits": 8
314
+ },
315
+ "model.layers.7.self_attn.k_proj": {
316
+ "group_size": 64,
317
+ "bits": 8
318
+ },
319
+ "model.layers.7.self_attn.v_proj": {
320
+ "group_size": 64,
321
+ "bits": 8
322
+ },
323
+ "model.layers.7.self_attn.o_proj": {
324
+ "group_size": 64,
325
+ "bits": 8
326
+ },
327
+ "model.layers.8.self_attn.q_proj": {
328
+ "group_size": 64,
329
+ "bits": 8
330
+ },
331
+ "model.layers.8.self_attn.k_proj": {
332
+ "group_size": 64,
333
+ "bits": 8
334
+ },
335
+ "model.layers.8.self_attn.v_proj": {
336
+ "group_size": 64,
337
+ "bits": 8
338
+ },
339
+ "model.layers.8.self_attn.o_proj": {
340
+ "group_size": 64,
341
+ "bits": 8
342
+ },
343
+ "model.layers.9.self_attn.q_proj": {
344
+ "group_size": 64,
345
+ "bits": 8
346
+ },
347
+ "model.layers.9.self_attn.k_proj": {
348
+ "group_size": 64,
349
+ "bits": 8
350
+ },
351
+ "model.layers.9.self_attn.v_proj": {
352
+ "group_size": 64,
353
+ "bits": 8
354
+ },
355
+ "model.layers.9.self_attn.o_proj": {
356
+ "group_size": 64,
357
+ "bits": 8
358
+ },
359
+ "model.layers.10.self_attn.q_proj": {
360
+ "group_size": 64,
361
+ "bits": 8
362
+ },
363
+ "model.layers.10.self_attn.k_proj": {
364
+ "group_size": 64,
365
+ "bits": 8
366
+ },
367
+ "model.layers.10.self_attn.v_proj": {
368
+ "group_size": 64,
369
+ "bits": 8
370
+ },
371
+ "model.layers.10.self_attn.o_proj": {
372
+ "group_size": 64,
373
+ "bits": 8
374
+ },
375
+ "model.layers.11.self_attn.q_proj": {
376
+ "group_size": 64,
377
+ "bits": 8
378
+ },
379
+ "model.layers.11.self_attn.k_proj": {
380
+ "group_size": 64,
381
+ "bits": 8
382
+ },
383
+ "model.layers.11.self_attn.v_proj": {
384
+ "group_size": 64,
385
+ "bits": 8
386
+ },
387
+ "model.layers.11.self_attn.o_proj": {
388
+ "group_size": 64,
389
+ "bits": 8
390
+ },
391
+ "model.layers.12.self_attn.q_proj": {
392
+ "group_size": 64,
393
+ "bits": 8
394
+ },
395
+ "model.layers.12.self_attn.k_proj": {
396
+ "group_size": 64,
397
+ "bits": 8
398
+ },
399
+ "model.layers.12.self_attn.v_proj": {
400
+ "group_size": 64,
401
+ "bits": 8
402
+ },
403
+ "model.layers.12.self_attn.o_proj": {
404
+ "group_size": 64,
405
+ "bits": 8
406
+ },
407
+ "model.layers.13.self_attn.q_proj": {
408
+ "group_size": 64,
409
+ "bits": 8
410
+ },
411
+ "model.layers.13.self_attn.k_proj": {
412
+ "group_size": 64,
413
+ "bits": 8
414
+ },
415
+ "model.layers.13.self_attn.v_proj": {
416
+ "group_size": 64,
417
+ "bits": 8
418
+ },
419
+ "model.layers.13.self_attn.o_proj": {
420
+ "group_size": 64,
421
+ "bits": 8
422
+ },
423
+ "model.layers.14.self_attn.q_proj": {
424
+ "group_size": 64,
425
+ "bits": 8
426
+ },
427
+ "model.layers.14.self_attn.k_proj": {
428
+ "group_size": 64,
429
+ "bits": 8
430
+ },
431
+ "model.layers.14.self_attn.v_proj": {
432
+ "group_size": 64,
433
+ "bits": 8
434
+ },
435
+ "model.layers.14.self_attn.o_proj": {
436
+ "group_size": 64,
437
+ "bits": 8
438
+ },
439
+ "model.layers.15.self_attn.q_proj": {
440
+ "group_size": 64,
441
+ "bits": 8
442
+ },
443
+ "model.layers.15.self_attn.k_proj": {
444
+ "group_size": 64,
445
+ "bits": 8
446
+ },
447
+ "model.layers.15.self_attn.v_proj": {
448
+ "group_size": 64,
449
+ "bits": 8
450
+ },
451
+ "model.layers.15.self_attn.o_proj": {
452
+ "group_size": 64,
453
+ "bits": 8
454
+ },
455
+ "model.layers.16.self_attn.q_proj": {
456
+ "group_size": 64,
457
+ "bits": 8
458
+ },
459
+ "model.layers.16.self_attn.k_proj": {
460
+ "group_size": 64,
461
+ "bits": 8
462
+ },
463
+ "model.layers.16.self_attn.v_proj": {
464
+ "group_size": 64,
465
+ "bits": 8
466
+ },
467
+ "model.layers.16.self_attn.o_proj": {
468
+ "group_size": 64,
469
+ "bits": 8
470
+ },
471
+ "model.layers.17.self_attn.q_proj": {
472
+ "group_size": 64,
473
+ "bits": 8
474
+ },
475
+ "model.layers.17.self_attn.k_proj": {
476
+ "group_size": 64,
477
+ "bits": 8
478
+ },
479
+ "model.layers.17.self_attn.v_proj": {
480
+ "group_size": 64,
481
+ "bits": 8
482
+ },
483
+ "model.layers.17.self_attn.o_proj": {
484
+ "group_size": 64,
485
+ "bits": 8
486
+ },
487
+ "model.layers.18.self_attn.q_proj": {
488
+ "group_size": 64,
489
+ "bits": 8
490
+ },
491
+ "model.layers.18.self_attn.k_proj": {
492
+ "group_size": 64,
493
+ "bits": 8
494
+ },
495
+ "model.layers.18.self_attn.v_proj": {
496
+ "group_size": 64,
497
+ "bits": 8
498
+ },
499
+ "model.layers.18.self_attn.o_proj": {
500
+ "group_size": 64,
501
+ "bits": 8
502
+ },
503
+ "model.layers.19.self_attn.q_proj": {
504
+ "group_size": 64,
505
+ "bits": 8
506
+ },
507
+ "model.layers.19.self_attn.k_proj": {
508
+ "group_size": 64,
509
+ "bits": 8
510
+ },
511
+ "model.layers.19.self_attn.v_proj": {
512
+ "group_size": 64,
513
+ "bits": 8
514
+ },
515
+ "model.layers.19.self_attn.o_proj": {
516
+ "group_size": 64,
517
+ "bits": 8
518
+ },
519
+ "model.layers.20.self_attn.q_proj": {
520
+ "group_size": 64,
521
+ "bits": 8
522
+ },
523
+ "model.layers.20.self_attn.k_proj": {
524
+ "group_size": 64,
525
+ "bits": 8
526
+ },
527
+ "model.layers.20.self_attn.v_proj": {
528
+ "group_size": 64,
529
+ "bits": 8
530
+ },
531
+ "model.layers.20.self_attn.o_proj": {
532
+ "group_size": 64,
533
+ "bits": 8
534
+ },
535
+ "model.layers.21.self_attn.q_proj": {
536
+ "group_size": 64,
537
+ "bits": 8
538
+ },
539
+ "model.layers.21.self_attn.k_proj": {
540
+ "group_size": 64,
541
+ "bits": 8
542
+ },
543
+ "model.layers.21.self_attn.v_proj": {
544
+ "group_size": 64,
545
+ "bits": 8
546
+ },
547
+ "model.layers.21.self_attn.o_proj": {
548
+ "group_size": 64,
549
+ "bits": 8
550
+ },
551
+ "model.layers.22.self_attn.q_proj": {
552
+ "group_size": 64,
553
+ "bits": 8
554
+ },
555
+ "model.layers.22.self_attn.k_proj": {
556
+ "group_size": 64,
557
+ "bits": 8
558
+ },
559
+ "model.layers.22.self_attn.v_proj": {
560
+ "group_size": 64,
561
+ "bits": 8
562
+ },
563
+ "model.layers.22.self_attn.o_proj": {
564
+ "group_size": 64,
565
+ "bits": 8
566
+ },
567
+ "model.layers.23.self_attn.q_proj": {
568
+ "group_size": 64,
569
+ "bits": 8
570
+ },
571
+ "model.layers.23.self_attn.k_proj": {
572
+ "group_size": 64,
573
+ "bits": 8
574
+ },
575
+ "model.layers.23.self_attn.v_proj": {
576
+ "group_size": 64,
577
+ "bits": 8
578
+ },
579
+ "model.layers.23.self_attn.o_proj": {
580
+ "group_size": 64,
581
+ "bits": 8
582
+ },
583
+ "model.layers.24.self_attn.q_proj": {
584
+ "group_size": 64,
585
+ "bits": 8
586
+ },
587
+ "model.layers.24.self_attn.k_proj": {
588
+ "group_size": 64,
589
+ "bits": 8
590
+ },
591
+ "model.layers.24.self_attn.v_proj": {
592
+ "group_size": 64,
593
+ "bits": 8
594
+ },
595
+ "model.layers.24.self_attn.o_proj": {
596
+ "group_size": 64,
597
+ "bits": 8
598
+ },
599
+ "model.layers.25.self_attn.q_proj": {
600
+ "group_size": 64,
601
+ "bits": 8
602
+ },
603
+ "model.layers.25.self_attn.k_proj": {
604
+ "group_size": 64,
605
+ "bits": 8
606
+ },
607
+ "model.layers.25.self_attn.v_proj": {
608
+ "group_size": 64,
609
+ "bits": 8
610
+ },
611
+ "model.layers.25.self_attn.o_proj": {
612
+ "group_size": 64,
613
+ "bits": 8
614
+ },
615
+ "model.layers.26.self_attn.q_proj": {
616
+ "group_size": 64,
617
+ "bits": 8
618
+ },
619
+ "model.layers.26.self_attn.k_proj": {
620
+ "group_size": 64,
621
+ "bits": 8
622
+ },
623
+ "model.layers.26.self_attn.v_proj": {
624
+ "group_size": 64,
625
+ "bits": 8
626
+ },
627
+ "model.layers.26.self_attn.o_proj": {
628
+ "group_size": 64,
629
+ "bits": 8
630
+ },
631
+ "model.layers.27.self_attn.q_proj": {
632
+ "group_size": 64,
633
+ "bits": 8
634
+ },
635
+ "model.layers.27.self_attn.k_proj": {
636
+ "group_size": 64,
637
+ "bits": 8
638
+ },
639
+ "model.layers.27.self_attn.v_proj": {
640
+ "group_size": 64,
641
+ "bits": 8
642
+ },
643
+ "model.layers.27.self_attn.o_proj": {
644
+ "group_size": 64,
645
+ "bits": 8
646
+ },
647
+ "model.layers.28.self_attn.q_proj": {
648
+ "group_size": 64,
649
+ "bits": 8
650
+ },
651
+ "model.layers.28.self_attn.k_proj": {
652
+ "group_size": 64,
653
+ "bits": 8
654
+ },
655
+ "model.layers.28.self_attn.v_proj": {
656
+ "group_size": 64,
657
+ "bits": 8
658
+ },
659
+ "model.layers.28.self_attn.o_proj": {
660
+ "group_size": 64,
661
+ "bits": 8
662
+ },
663
+ "model.layers.29.self_attn.q_proj": {
664
+ "group_size": 64,
665
+ "bits": 8
666
+ },
667
+ "model.layers.29.self_attn.k_proj": {
668
+ "group_size": 64,
669
+ "bits": 8
670
+ },
671
+ "model.layers.29.self_attn.v_proj": {
672
+ "group_size": 64,
673
+ "bits": 8
674
+ },
675
+ "model.layers.29.self_attn.o_proj": {
676
+ "group_size": 64,
677
+ "bits": 8
678
+ },
679
+ "model.layers.30.self_attn.q_proj": {
680
+ "group_size": 64,
681
+ "bits": 8
682
+ },
683
+ "model.layers.30.self_attn.k_proj": {
684
+ "group_size": 64,
685
+ "bits": 8
686
+ },
687
+ "model.layers.30.self_attn.v_proj": {
688
+ "group_size": 64,
689
+ "bits": 8
690
+ },
691
+ "model.layers.30.self_attn.o_proj": {
692
+ "group_size": 64,
693
+ "bits": 8
694
+ },
695
+ "model.layers.31.self_attn.q_proj": {
696
+ "group_size": 64,
697
+ "bits": 8
698
+ },
699
+ "model.layers.31.self_attn.k_proj": {
700
+ "group_size": 64,
701
+ "bits": 8
702
+ },
703
+ "model.layers.31.self_attn.v_proj": {
704
+ "group_size": 64,
705
+ "bits": 8
706
+ },
707
+ "model.layers.31.self_attn.o_proj": {
708
+ "group_size": 64,
709
+ "bits": 8
710
+ },
711
+ "model.layers.32.self_attn.q_proj": {
712
+ "group_size": 64,
713
+ "bits": 8
714
+ },
715
+ "model.layers.32.self_attn.k_proj": {
716
+ "group_size": 64,
717
+ "bits": 8
718
+ },
719
+ "model.layers.32.self_attn.v_proj": {
720
+ "group_size": 64,
721
+ "bits": 8
722
+ },
723
+ "model.layers.32.self_attn.o_proj": {
724
+ "group_size": 64,
725
+ "bits": 8
726
+ },
727
+ "model.layers.33.self_attn.q_proj": {
728
+ "group_size": 64,
729
+ "bits": 8
730
+ },
731
+ "model.layers.33.self_attn.k_proj": {
732
+ "group_size": 64,
733
+ "bits": 8
734
+ },
735
+ "model.layers.33.self_attn.v_proj": {
736
+ "group_size": 64,
737
+ "bits": 8
738
+ },
739
+ "model.layers.33.self_attn.o_proj": {
740
+ "group_size": 64,
741
+ "bits": 8
742
+ },
743
+ "model.layers.34.self_attn.q_proj": {
744
+ "group_size": 64,
745
+ "bits": 8
746
+ },
747
+ "model.layers.34.self_attn.k_proj": {
748
+ "group_size": 64,
749
+ "bits": 8
750
+ },
751
+ "model.layers.34.self_attn.v_proj": {
752
+ "group_size": 64,
753
+ "bits": 8
754
+ },
755
+ "model.layers.34.self_attn.o_proj": {
756
+ "group_size": 64,
757
+ "bits": 8
758
+ },
759
+ "model.layers.35.self_attn.q_proj": {
760
+ "group_size": 64,
761
+ "bits": 8
762
+ },
763
+ "model.layers.35.self_attn.k_proj": {
764
+ "group_size": 64,
765
+ "bits": 8
766
+ },
767
+ "model.layers.35.self_attn.v_proj": {
768
+ "group_size": 64,
769
+ "bits": 8
770
+ },
771
+ "model.layers.35.self_attn.o_proj": {
772
+ "group_size": 64,
773
+ "bits": 8
774
+ },
775
+ "model.layers.36.self_attn.q_proj": {
776
+ "group_size": 64,
777
+ "bits": 8
778
+ },
779
+ "model.layers.36.self_attn.k_proj": {
780
+ "group_size": 64,
781
+ "bits": 8
782
+ },
783
+ "model.layers.36.self_attn.v_proj": {
784
+ "group_size": 64,
785
+ "bits": 8
786
+ },
787
+ "model.layers.36.self_attn.o_proj": {
788
+ "group_size": 64,
789
+ "bits": 8
790
+ },
791
+ "model.layers.37.self_attn.q_proj": {
792
+ "group_size": 64,
793
+ "bits": 8
794
+ },
795
+ "model.layers.37.self_attn.k_proj": {
796
+ "group_size": 64,
797
+ "bits": 8
798
+ },
799
+ "model.layers.37.self_attn.v_proj": {
800
+ "group_size": 64,
801
+ "bits": 8
802
+ },
803
+ "model.layers.37.self_attn.o_proj": {
804
+ "group_size": 64,
805
+ "bits": 8
806
+ },
807
+ "model.layers.38.self_attn.q_proj": {
808
+ "group_size": 64,
809
+ "bits": 8
810
+ },
811
+ "model.layers.38.self_attn.k_proj": {
812
+ "group_size": 64,
813
+ "bits": 8
814
+ },
815
+ "model.layers.38.self_attn.v_proj": {
816
+ "group_size": 64,
817
+ "bits": 8
818
+ },
819
+ "model.layers.38.self_attn.o_proj": {
820
+ "group_size": 64,
821
+ "bits": 8
822
+ },
823
+ "model.layers.39.self_attn.q_proj": {
824
+ "group_size": 64,
825
+ "bits": 8
826
+ },
827
+ "model.layers.39.self_attn.k_proj": {
828
+ "group_size": 64,
829
+ "bits": 8
830
+ },
831
+ "model.layers.39.self_attn.v_proj": {
832
+ "group_size": 64,
833
+ "bits": 8
834
+ },
835
+ "model.layers.39.self_attn.o_proj": {
836
+ "group_size": 64,
837
+ "bits": 8
838
+ },
839
+ "model.layers.40.self_attn.q_proj": {
840
+ "group_size": 64,
841
+ "bits": 8
842
+ },
843
+ "model.layers.40.self_attn.k_proj": {
844
+ "group_size": 64,
845
+ "bits": 8
846
+ },
847
+ "model.layers.40.self_attn.v_proj": {
848
+ "group_size": 64,
849
+ "bits": 8
850
+ },
851
+ "model.layers.40.self_attn.o_proj": {
852
+ "group_size": 64,
853
+ "bits": 8
854
+ },
855
+ "model.layers.41.self_attn.q_proj": {
856
+ "group_size": 64,
857
+ "bits": 8
858
+ },
859
+ "model.layers.41.self_attn.k_proj": {
860
+ "group_size": 64,
861
+ "bits": 8
862
+ },
863
+ "model.layers.41.self_attn.v_proj": {
864
+ "group_size": 64,
865
+ "bits": 8
866
+ },
867
+ "model.layers.41.self_attn.o_proj": {
868
+ "group_size": 64,
869
+ "bits": 8
870
+ },
871
+ "model.layers.42.self_attn.q_proj": {
872
+ "group_size": 64,
873
+ "bits": 8
874
+ },
875
+ "model.layers.42.self_attn.k_proj": {
876
+ "group_size": 64,
877
+ "bits": 8
878
+ },
879
+ "model.layers.42.self_attn.v_proj": {
880
+ "group_size": 64,
881
+ "bits": 8
882
+ },
883
+ "model.layers.42.self_attn.o_proj": {
884
+ "group_size": 64,
885
+ "bits": 8
886
+ },
887
+ "model.layers.43.self_attn.q_proj": {
888
+ "group_size": 64,
889
+ "bits": 8
890
+ },
891
+ "model.layers.43.self_attn.k_proj": {
892
+ "group_size": 64,
893
+ "bits": 8
894
+ },
895
+ "model.layers.43.self_attn.v_proj": {
896
+ "group_size": 64,
897
+ "bits": 8
898
+ },
899
+ "model.layers.43.self_attn.o_proj": {
900
+ "group_size": 64,
901
+ "bits": 8
902
+ },
903
+ "model.layers.44.self_attn.q_proj": {
904
+ "group_size": 64,
905
+ "bits": 8
906
+ },
907
+ "model.layers.44.self_attn.k_proj": {
908
+ "group_size": 64,
909
+ "bits": 8
910
+ },
911
+ "model.layers.44.self_attn.v_proj": {
912
+ "group_size": 64,
913
+ "bits": 8
914
+ },
915
+ "model.layers.44.self_attn.o_proj": {
916
+ "group_size": 64,
917
+ "bits": 8
918
+ },
919
+ "model.layers.45.self_attn.q_proj": {
920
+ "group_size": 64,
921
+ "bits": 8
922
+ },
923
+ "model.layers.45.self_attn.k_proj": {
924
+ "group_size": 64,
925
+ "bits": 8
926
+ },
927
+ "model.layers.45.self_attn.v_proj": {
928
+ "group_size": 64,
929
+ "bits": 8
930
+ },
931
+ "model.layers.45.self_attn.o_proj": {
932
+ "group_size": 64,
933
+ "bits": 8
934
+ },
935
+ "model.layers.46.self_attn.q_proj": {
936
+ "group_size": 64,
937
+ "bits": 8
938
+ },
939
+ "model.layers.46.self_attn.k_proj": {
940
+ "group_size": 64,
941
+ "bits": 8
942
+ },
943
+ "model.layers.46.self_attn.v_proj": {
944
+ "group_size": 64,
945
+ "bits": 8
946
+ },
947
+ "model.layers.46.self_attn.o_proj": {
948
+ "group_size": 64,
949
+ "bits": 8
950
+ },
951
+ "model.layers.47.self_attn.q_proj": {
952
+ "group_size": 64,
953
+ "bits": 8
954
+ },
955
+ "model.layers.47.self_attn.k_proj": {
956
+ "group_size": 64,
957
+ "bits": 8
958
+ },
959
+ "model.layers.47.self_attn.v_proj": {
960
+ "group_size": 64,
961
+ "bits": 8
962
+ },
963
+ "model.layers.47.self_attn.o_proj": {
964
+ "group_size": 64,
965
+ "bits": 8
966
+ },
967
+ "model.layers.48.self_attn.q_proj": {
968
+ "group_size": 64,
969
+ "bits": 8
970
+ },
971
+ "model.layers.48.self_attn.k_proj": {
972
+ "group_size": 64,
973
+ "bits": 8
974
+ },
975
+ "model.layers.48.self_attn.v_proj": {
976
+ "group_size": 64,
977
+ "bits": 8
978
+ },
979
+ "model.layers.48.self_attn.o_proj": {
980
+ "group_size": 64,
981
+ "bits": 8
982
+ },
983
+ "model.layers.49.self_attn.q_proj": {
984
+ "group_size": 64,
985
+ "bits": 8
986
+ },
987
+ "model.layers.49.self_attn.k_proj": {
988
+ "group_size": 64,
989
+ "bits": 8
990
+ },
991
+ "model.layers.49.self_attn.v_proj": {
992
+ "group_size": 64,
993
+ "bits": 8
994
+ },
995
+ "model.layers.49.self_attn.o_proj": {
996
+ "group_size": 64,
997
+ "bits": 8
998
+ },
999
+ "model.layers.50.self_attn.q_proj": {
1000
+ "group_size": 64,
1001
+ "bits": 8
1002
+ },
1003
+ "model.layers.50.self_attn.k_proj": {
1004
+ "group_size": 64,
1005
+ "bits": 8
1006
+ },
1007
+ "model.layers.50.self_attn.v_proj": {
1008
+ "group_size": 64,
1009
+ "bits": 8
1010
+ },
1011
+ "model.layers.50.self_attn.o_proj": {
1012
+ "group_size": 64,
1013
+ "bits": 8
1014
+ },
1015
+ "model.layers.51.self_attn.q_proj": {
1016
+ "group_size": 64,
1017
+ "bits": 8
1018
+ },
1019
+ "model.layers.51.self_attn.k_proj": {
1020
+ "group_size": 64,
1021
+ "bits": 8
1022
+ },
1023
+ "model.layers.51.self_attn.v_proj": {
1024
+ "group_size": 64,
1025
+ "bits": 8
1026
+ },
1027
+ "model.layers.51.self_attn.o_proj": {
1028
+ "group_size": 64,
1029
+ "bits": 8
1030
+ },
1031
+ "model.layers.52.self_attn.q_proj": {
1032
+ "group_size": 64,
1033
+ "bits": 8
1034
+ },
1035
+ "model.layers.52.self_attn.k_proj": {
1036
+ "group_size": 64,
1037
+ "bits": 8
1038
+ },
1039
+ "model.layers.52.self_attn.v_proj": {
1040
+ "group_size": 64,
1041
+ "bits": 8
1042
+ },
1043
+ "model.layers.52.self_attn.o_proj": {
1044
+ "group_size": 64,
1045
+ "bits": 8
1046
+ },
1047
+ "model.layers.53.self_attn.q_proj": {
1048
+ "group_size": 64,
1049
+ "bits": 8
1050
+ },
1051
+ "model.layers.53.self_attn.k_proj": {
1052
+ "group_size": 64,
1053
+ "bits": 8
1054
+ },
1055
+ "model.layers.53.self_attn.v_proj": {
1056
+ "group_size": 64,
1057
+ "bits": 8
1058
+ },
1059
+ "model.layers.53.self_attn.o_proj": {
1060
+ "group_size": 64,
1061
+ "bits": 8
1062
+ },
1063
+ "model.layers.54.self_attn.q_proj": {
1064
+ "group_size": 64,
1065
+ "bits": 8
1066
+ },
1067
+ "model.layers.54.self_attn.k_proj": {
1068
+ "group_size": 64,
1069
+ "bits": 8
1070
+ },
1071
+ "model.layers.54.self_attn.v_proj": {
1072
+ "group_size": 64,
1073
+ "bits": 8
1074
+ },
1075
+ "model.layers.54.self_attn.o_proj": {
1076
+ "group_size": 64,
1077
+ "bits": 8
1078
+ },
1079
+ "model.layers.55.self_attn.q_proj": {
1080
+ "group_size": 64,
1081
+ "bits": 8
1082
+ },
1083
+ "model.layers.55.self_attn.k_proj": {
1084
+ "group_size": 64,
1085
+ "bits": 8
1086
+ },
1087
+ "model.layers.55.self_attn.v_proj": {
1088
+ "group_size": 64,
1089
+ "bits": 8
1090
+ },
1091
+ "model.layers.55.self_attn.o_proj": {
1092
+ "group_size": 64,
1093
+ "bits": 8
1094
+ },
1095
+ "model.layers.56.self_attn.q_proj": {
1096
+ "group_size": 64,
1097
+ "bits": 8
1098
+ },
1099
+ "model.layers.56.self_attn.k_proj": {
1100
+ "group_size": 64,
1101
+ "bits": 8
1102
+ },
1103
+ "model.layers.56.self_attn.v_proj": {
1104
+ "group_size": 64,
1105
+ "bits": 8
1106
+ },
1107
+ "model.layers.56.self_attn.o_proj": {
1108
+ "group_size": 64,
1109
+ "bits": 8
1110
+ },
1111
+ "model.layers.57.self_attn.q_proj": {
1112
+ "group_size": 64,
1113
+ "bits": 8
1114
+ },
1115
+ "model.layers.57.self_attn.k_proj": {
1116
+ "group_size": 64,
1117
+ "bits": 8
1118
+ },
1119
+ "model.layers.57.self_attn.v_proj": {
1120
+ "group_size": 64,
1121
+ "bits": 8
1122
+ },
1123
+ "model.layers.57.self_attn.o_proj": {
1124
+ "group_size": 64,
1125
+ "bits": 8
1126
+ },
1127
+ "model.layers.58.self_attn.q_proj": {
1128
+ "group_size": 64,
1129
+ "bits": 8
1130
+ },
1131
+ "model.layers.58.self_attn.k_proj": {
1132
+ "group_size": 64,
1133
+ "bits": 8
1134
+ },
1135
+ "model.layers.58.self_attn.v_proj": {
1136
+ "group_size": 64,
1137
+ "bits": 8
1138
+ },
1139
+ "model.layers.58.self_attn.o_proj": {
1140
+ "group_size": 64,
1141
+ "bits": 8
1142
+ },
1143
+ "model.layers.59.self_attn.q_proj": {
1144
+ "group_size": 64,
1145
+ "bits": 8
1146
+ },
1147
+ "model.layers.59.self_attn.k_proj": {
1148
+ "group_size": 64,
1149
+ "bits": 8
1150
+ },
1151
+ "model.layers.59.self_attn.v_proj": {
1152
+ "group_size": 64,
1153
+ "bits": 8
1154
+ },
1155
+ "model.layers.59.self_attn.o_proj": {
1156
+ "group_size": 64,
1157
+ "bits": 8
1158
+ },
1159
+ "model.layers.60.self_attn.q_proj": {
1160
+ "group_size": 64,
1161
+ "bits": 8
1162
+ },
1163
+ "model.layers.60.self_attn.k_proj": {
1164
+ "group_size": 64,
1165
+ "bits": 8
1166
+ },
1167
+ "model.layers.60.self_attn.v_proj": {
1168
+ "group_size": 64,
1169
+ "bits": 8
1170
+ },
1171
+ "model.layers.60.self_attn.o_proj": {
1172
+ "group_size": 64,
1173
+ "bits": 8
1174
+ },
1175
+ "model.layers.61.self_attn.q_proj": {
1176
+ "group_size": 64,
1177
+ "bits": 8
1178
+ },
1179
+ "model.layers.61.self_attn.k_proj": {
1180
+ "group_size": 64,
1181
+ "bits": 8
1182
+ },
1183
+ "model.layers.61.self_attn.v_proj": {
1184
+ "group_size": 64,
1185
+ "bits": 8
1186
+ },
1187
+ "model.layers.61.self_attn.o_proj": {
1188
+ "group_size": 64,
1189
+ "bits": 8
1190
+ },
1191
+ "model.layers.62.self_attn.q_proj": {
1192
+ "group_size": 64,
1193
+ "bits": 8
1194
+ },
1195
+ "model.layers.62.self_attn.k_proj": {
1196
+ "group_size": 64,
1197
+ "bits": 8
1198
+ },
1199
+ "model.layers.62.self_attn.v_proj": {
1200
+ "group_size": 64,
1201
+ "bits": 8
1202
+ },
1203
+ "model.layers.62.self_attn.o_proj": {
1204
+ "group_size": 64,
1205
+ "bits": 8
1206
+ },
1207
+ "model.layers.63.self_attn.q_proj": {
1208
+ "group_size": 64,
1209
+ "bits": 8
1210
+ },
1211
+ "model.layers.63.self_attn.k_proj": {
1212
+ "group_size": 64,
1213
+ "bits": 8
1214
+ },
1215
+ "model.layers.63.self_attn.v_proj": {
1216
+ "group_size": 64,
1217
+ "bits": 8
1218
+ },
1219
+ "model.layers.63.self_attn.o_proj": {
1220
+ "group_size": 64,
1221
+ "bits": 8
1222
+ },
1223
+ "model.layers.64.self_attn.q_proj": {
1224
+ "group_size": 64,
1225
+ "bits": 8
1226
+ },
1227
+ "model.layers.64.self_attn.k_proj": {
1228
+ "group_size": 64,
1229
+ "bits": 8
1230
+ },
1231
+ "model.layers.64.self_attn.v_proj": {
1232
+ "group_size": 64,
1233
+ "bits": 8
1234
+ },
1235
+ "model.layers.64.self_attn.o_proj": {
1236
+ "group_size": 64,
1237
+ "bits": 8
1238
+ },
1239
+ "model.layers.65.self_attn.q_proj": {
1240
+ "group_size": 64,
1241
+ "bits": 8
1242
+ },
1243
+ "model.layers.65.self_attn.k_proj": {
1244
+ "group_size": 64,
1245
+ "bits": 8
1246
+ },
1247
+ "model.layers.65.self_attn.v_proj": {
1248
+ "group_size": 64,
1249
+ "bits": 8
1250
+ },
1251
+ "model.layers.65.self_attn.o_proj": {
1252
+ "group_size": 64,
1253
+ "bits": 8
1254
+ },
1255
+ "model.layers.66.self_attn.q_proj": {
1256
+ "group_size": 64,
1257
+ "bits": 8
1258
+ },
1259
+ "model.layers.66.self_attn.k_proj": {
1260
+ "group_size": 64,
1261
+ "bits": 8
1262
+ },
1263
+ "model.layers.66.self_attn.v_proj": {
1264
+ "group_size": 64,
1265
+ "bits": 8
1266
+ },
1267
+ "model.layers.66.self_attn.o_proj": {
1268
+ "group_size": 64,
1269
+ "bits": 8
1270
+ },
1271
+ "model.layers.67.self_attn.q_proj": {
1272
+ "group_size": 64,
1273
+ "bits": 8
1274
+ },
1275
+ "model.layers.67.self_attn.k_proj": {
1276
+ "group_size": 64,
1277
+ "bits": 8
1278
+ },
1279
+ "model.layers.67.self_attn.v_proj": {
1280
+ "group_size": 64,
1281
+ "bits": 8
1282
+ },
1283
+ "model.layers.67.self_attn.o_proj": {
1284
+ "group_size": 64,
1285
+ "bits": 8
1286
+ },
1287
+ "model.layers.68.self_attn.q_proj": {
1288
+ "group_size": 64,
1289
+ "bits": 8
1290
+ },
1291
+ "model.layers.68.self_attn.k_proj": {
1292
+ "group_size": 64,
1293
+ "bits": 8
1294
+ },
1295
+ "model.layers.68.self_attn.v_proj": {
1296
+ "group_size": 64,
1297
+ "bits": 8
1298
+ },
1299
+ "model.layers.68.self_attn.o_proj": {
1300
+ "group_size": 64,
1301
+ "bits": 8
1302
+ },
1303
+ "model.layers.69.self_attn.q_proj": {
1304
+ "group_size": 64,
1305
+ "bits": 8
1306
+ },
1307
+ "model.layers.69.self_attn.k_proj": {
1308
+ "group_size": 64,
1309
+ "bits": 8
1310
+ },
1311
+ "model.layers.69.self_attn.v_proj": {
1312
+ "group_size": 64,
1313
+ "bits": 8
1314
+ },
1315
+ "model.layers.69.self_attn.o_proj": {
1316
+ "group_size": 64,
1317
+ "bits": 8
1318
+ },
1319
+ "lm_head": {
1320
+ "group_size": 32,
1321
+ "bits": 8
1322
+ }
1323
+ },
1324
+ "quantization_config": {
1325
+ "group_size": 64,
1326
+ "bits": 6,
1327
+ "mode": "affine",
1328
+ "model.embed_tokens": {
1329
+ "group_size": 32,
1330
+ "bits": 8
1331
+ },
1332
+ "model.layers.0.self_attn.q_proj": {
1333
+ "group_size": 64,
1334
+ "bits": 8
1335
+ },
1336
+ "model.layers.0.self_attn.k_proj": {
1337
+ "group_size": 64,
1338
+ "bits": 8
1339
+ },
1340
+ "model.layers.0.self_attn.v_proj": {
1341
+ "group_size": 64,
1342
+ "bits": 8
1343
+ },
1344
+ "model.layers.0.self_attn.o_proj": {
1345
+ "group_size": 64,
1346
+ "bits": 8
1347
+ },
1348
+ "model.layers.0.mlp.gate_proj": {
1349
+ "group_size": 64,
1350
+ "bits": 8
1351
+ },
1352
+ "model.layers.0.mlp.up_proj": {
1353
+ "group_size": 64,
1354
+ "bits": 8
1355
+ },
1356
+ "model.layers.0.mlp.down_proj": {
1357
+ "group_size": 64,
1358
+ "bits": 8
1359
+ },
1360
+ "model.layers.1.self_attn.q_proj": {
1361
+ "group_size": 64,
1362
+ "bits": 8
1363
+ },
1364
+ "model.layers.1.self_attn.k_proj": {
1365
+ "group_size": 64,
1366
+ "bits": 8
1367
+ },
1368
+ "model.layers.1.self_attn.v_proj": {
1369
+ "group_size": 64,
1370
+ "bits": 8
1371
+ },
1372
+ "model.layers.1.self_attn.o_proj": {
1373
+ "group_size": 64,
1374
+ "bits": 8
1375
+ },
1376
+ "model.layers.2.self_attn.q_proj": {
1377
+ "group_size": 64,
1378
+ "bits": 8
1379
+ },
1380
+ "model.layers.2.self_attn.k_proj": {
1381
+ "group_size": 64,
1382
+ "bits": 8
1383
+ },
1384
+ "model.layers.2.self_attn.v_proj": {
1385
+ "group_size": 64,
1386
+ "bits": 8
1387
+ },
1388
+ "model.layers.2.self_attn.o_proj": {
1389
+ "group_size": 64,
1390
+ "bits": 8
1391
+ },
1392
+ "model.layers.3.self_attn.q_proj": {
1393
+ "group_size": 64,
1394
+ "bits": 8
1395
+ },
1396
+ "model.layers.3.self_attn.k_proj": {
1397
+ "group_size": 64,
1398
+ "bits": 8
1399
+ },
1400
+ "model.layers.3.self_attn.v_proj": {
1401
+ "group_size": 64,
1402
+ "bits": 8
1403
+ },
1404
+ "model.layers.3.self_attn.o_proj": {
1405
+ "group_size": 64,
1406
+ "bits": 8
1407
+ },
1408
+ "model.layers.4.self_attn.q_proj": {
1409
+ "group_size": 64,
1410
+ "bits": 8
1411
+ },
1412
+ "model.layers.4.self_attn.k_proj": {
1413
+ "group_size": 64,
1414
+ "bits": 8
1415
+ },
1416
+ "model.layers.4.self_attn.v_proj": {
1417
+ "group_size": 64,
1418
+ "bits": 8
1419
+ },
1420
+ "model.layers.4.self_attn.o_proj": {
1421
+ "group_size": 64,
1422
+ "bits": 8
1423
+ },
1424
+ "model.layers.5.self_attn.q_proj": {
1425
+ "group_size": 64,
1426
+ "bits": 8
1427
+ },
1428
+ "model.layers.5.self_attn.k_proj": {
1429
+ "group_size": 64,
1430
+ "bits": 8
1431
+ },
1432
+ "model.layers.5.self_attn.v_proj": {
1433
+ "group_size": 64,
1434
+ "bits": 8
1435
+ },
1436
+ "model.layers.5.self_attn.o_proj": {
1437
+ "group_size": 64,
1438
+ "bits": 8
1439
+ },
1440
+ "model.layers.6.self_attn.q_proj": {
1441
+ "group_size": 64,
1442
+ "bits": 8
1443
+ },
1444
+ "model.layers.6.self_attn.k_proj": {
1445
+ "group_size": 64,
1446
+ "bits": 8
1447
+ },
1448
+ "model.layers.6.self_attn.v_proj": {
1449
+ "group_size": 64,
1450
+ "bits": 8
1451
+ },
1452
+ "model.layers.6.self_attn.o_proj": {
1453
+ "group_size": 64,
1454
+ "bits": 8
1455
+ },
1456
+ "model.layers.7.self_attn.q_proj": {
1457
+ "group_size": 64,
1458
+ "bits": 8
1459
+ },
1460
+ "model.layers.7.self_attn.k_proj": {
1461
+ "group_size": 64,
1462
+ "bits": 8
1463
+ },
1464
+ "model.layers.7.self_attn.v_proj": {
1465
+ "group_size": 64,
1466
+ "bits": 8
1467
+ },
1468
+ "model.layers.7.self_attn.o_proj": {
1469
+ "group_size": 64,
1470
+ "bits": 8
1471
+ },
1472
+ "model.layers.8.self_attn.q_proj": {
1473
+ "group_size": 64,
1474
+ "bits": 8
1475
+ },
1476
+ "model.layers.8.self_attn.k_proj": {
1477
+ "group_size": 64,
1478
+ "bits": 8
1479
+ },
1480
+ "model.layers.8.self_attn.v_proj": {
1481
+ "group_size": 64,
1482
+ "bits": 8
1483
+ },
1484
+ "model.layers.8.self_attn.o_proj": {
1485
+ "group_size": 64,
1486
+ "bits": 8
1487
+ },
1488
+ "model.layers.9.self_attn.q_proj": {
1489
+ "group_size": 64,
1490
+ "bits": 8
1491
+ },
1492
+ "model.layers.9.self_attn.k_proj": {
1493
+ "group_size": 64,
1494
+ "bits": 8
1495
+ },
1496
+ "model.layers.9.self_attn.v_proj": {
1497
+ "group_size": 64,
1498
+ "bits": 8
1499
+ },
1500
+ "model.layers.9.self_attn.o_proj": {
1501
+ "group_size": 64,
1502
+ "bits": 8
1503
+ },
1504
+ "model.layers.10.self_attn.q_proj": {
1505
+ "group_size": 64,
1506
+ "bits": 8
1507
+ },
1508
+ "model.layers.10.self_attn.k_proj": {
1509
+ "group_size": 64,
1510
+ "bits": 8
1511
+ },
1512
+ "model.layers.10.self_attn.v_proj": {
1513
+ "group_size": 64,
1514
+ "bits": 8
1515
+ },
1516
+ "model.layers.10.self_attn.o_proj": {
1517
+ "group_size": 64,
1518
+ "bits": 8
1519
+ },
1520
+ "model.layers.11.self_attn.q_proj": {
1521
+ "group_size": 64,
1522
+ "bits": 8
1523
+ },
1524
+ "model.layers.11.self_attn.k_proj": {
1525
+ "group_size": 64,
1526
+ "bits": 8
1527
+ },
1528
+ "model.layers.11.self_attn.v_proj": {
1529
+ "group_size": 64,
1530
+ "bits": 8
1531
+ },
1532
+ "model.layers.11.self_attn.o_proj": {
1533
+ "group_size": 64,
1534
+ "bits": 8
1535
+ },
1536
+ "model.layers.12.self_attn.q_proj": {
1537
+ "group_size": 64,
1538
+ "bits": 8
1539
+ },
1540
+ "model.layers.12.self_attn.k_proj": {
1541
+ "group_size": 64,
1542
+ "bits": 8
1543
+ },
1544
+ "model.layers.12.self_attn.v_proj": {
1545
+ "group_size": 64,
1546
+ "bits": 8
1547
+ },
1548
+ "model.layers.12.self_attn.o_proj": {
1549
+ "group_size": 64,
1550
+ "bits": 8
1551
+ },
1552
+ "model.layers.13.self_attn.q_proj": {
1553
+ "group_size": 64,
1554
+ "bits": 8
1555
+ },
1556
+ "model.layers.13.self_attn.k_proj": {
1557
+ "group_size": 64,
1558
+ "bits": 8
1559
+ },
1560
+ "model.layers.13.self_attn.v_proj": {
1561
+ "group_size": 64,
1562
+ "bits": 8
1563
+ },
1564
+ "model.layers.13.self_attn.o_proj": {
1565
+ "group_size": 64,
1566
+ "bits": 8
1567
+ },
1568
+ "model.layers.14.self_attn.q_proj": {
1569
+ "group_size": 64,
1570
+ "bits": 8
1571
+ },
1572
+ "model.layers.14.self_attn.k_proj": {
1573
+ "group_size": 64,
1574
+ "bits": 8
1575
+ },
1576
+ "model.layers.14.self_attn.v_proj": {
1577
+ "group_size": 64,
1578
+ "bits": 8
1579
+ },
1580
+ "model.layers.14.self_attn.o_proj": {
1581
+ "group_size": 64,
1582
+ "bits": 8
1583
+ },
1584
+ "model.layers.15.self_attn.q_proj": {
1585
+ "group_size": 64,
1586
+ "bits": 8
1587
+ },
1588
+ "model.layers.15.self_attn.k_proj": {
1589
+ "group_size": 64,
1590
+ "bits": 8
1591
+ },
1592
+ "model.layers.15.self_attn.v_proj": {
1593
+ "group_size": 64,
1594
+ "bits": 8
1595
+ },
1596
+ "model.layers.15.self_attn.o_proj": {
1597
+ "group_size": 64,
1598
+ "bits": 8
1599
+ },
1600
+ "model.layers.16.self_attn.q_proj": {
1601
+ "group_size": 64,
1602
+ "bits": 8
1603
+ },
1604
+ "model.layers.16.self_attn.k_proj": {
1605
+ "group_size": 64,
1606
+ "bits": 8
1607
+ },
1608
+ "model.layers.16.self_attn.v_proj": {
1609
+ "group_size": 64,
1610
+ "bits": 8
1611
+ },
1612
+ "model.layers.16.self_attn.o_proj": {
1613
+ "group_size": 64,
1614
+ "bits": 8
1615
+ },
1616
+ "model.layers.17.self_attn.q_proj": {
1617
+ "group_size": 64,
1618
+ "bits": 8
1619
+ },
1620
+ "model.layers.17.self_attn.k_proj": {
1621
+ "group_size": 64,
1622
+ "bits": 8
1623
+ },
1624
+ "model.layers.17.self_attn.v_proj": {
1625
+ "group_size": 64,
1626
+ "bits": 8
1627
+ },
1628
+ "model.layers.17.self_attn.o_proj": {
1629
+ "group_size": 64,
1630
+ "bits": 8
1631
+ },
1632
+ "model.layers.18.self_attn.q_proj": {
1633
+ "group_size": 64,
1634
+ "bits": 8
1635
+ },
1636
+ "model.layers.18.self_attn.k_proj": {
1637
+ "group_size": 64,
1638
+ "bits": 8
1639
+ },
1640
+ "model.layers.18.self_attn.v_proj": {
1641
+ "group_size": 64,
1642
+ "bits": 8
1643
+ },
1644
+ "model.layers.18.self_attn.o_proj": {
1645
+ "group_size": 64,
1646
+ "bits": 8
1647
+ },
1648
+ "model.layers.19.self_attn.q_proj": {
1649
+ "group_size": 64,
1650
+ "bits": 8
1651
+ },
1652
+ "model.layers.19.self_attn.k_proj": {
1653
+ "group_size": 64,
1654
+ "bits": 8
1655
+ },
1656
+ "model.layers.19.self_attn.v_proj": {
1657
+ "group_size": 64,
1658
+ "bits": 8
1659
+ },
1660
+ "model.layers.19.self_attn.o_proj": {
1661
+ "group_size": 64,
1662
+ "bits": 8
1663
+ },
1664
+ "model.layers.20.self_attn.q_proj": {
1665
+ "group_size": 64,
1666
+ "bits": 8
1667
+ },
1668
+ "model.layers.20.self_attn.k_proj": {
1669
+ "group_size": 64,
1670
+ "bits": 8
1671
+ },
1672
+ "model.layers.20.self_attn.v_proj": {
1673
+ "group_size": 64,
1674
+ "bits": 8
1675
+ },
1676
+ "model.layers.20.self_attn.o_proj": {
1677
+ "group_size": 64,
1678
+ "bits": 8
1679
+ },
1680
+ "model.layers.21.self_attn.q_proj": {
1681
+ "group_size": 64,
1682
+ "bits": 8
1683
+ },
1684
+ "model.layers.21.self_attn.k_proj": {
1685
+ "group_size": 64,
1686
+ "bits": 8
1687
+ },
1688
+ "model.layers.21.self_attn.v_proj": {
1689
+ "group_size": 64,
1690
+ "bits": 8
1691
+ },
1692
+ "model.layers.21.self_attn.o_proj": {
1693
+ "group_size": 64,
1694
+ "bits": 8
1695
+ },
1696
+ "model.layers.22.self_attn.q_proj": {
1697
+ "group_size": 64,
1698
+ "bits": 8
1699
+ },
1700
+ "model.layers.22.self_attn.k_proj": {
1701
+ "group_size": 64,
1702
+ "bits": 8
1703
+ },
1704
+ "model.layers.22.self_attn.v_proj": {
1705
+ "group_size": 64,
1706
+ "bits": 8
1707
+ },
1708
+ "model.layers.22.self_attn.o_proj": {
1709
+ "group_size": 64,
1710
+ "bits": 8
1711
+ },
1712
+ "model.layers.23.self_attn.q_proj": {
1713
+ "group_size": 64,
1714
+ "bits": 8
1715
+ },
1716
+ "model.layers.23.self_attn.k_proj": {
1717
+ "group_size": 64,
1718
+ "bits": 8
1719
+ },
1720
+ "model.layers.23.self_attn.v_proj": {
1721
+ "group_size": 64,
1722
+ "bits": 8
1723
+ },
1724
+ "model.layers.23.self_attn.o_proj": {
1725
+ "group_size": 64,
1726
+ "bits": 8
1727
+ },
1728
+ "model.layers.24.self_attn.q_proj": {
1729
+ "group_size": 64,
1730
+ "bits": 8
1731
+ },
1732
+ "model.layers.24.self_attn.k_proj": {
1733
+ "group_size": 64,
1734
+ "bits": 8
1735
+ },
1736
+ "model.layers.24.self_attn.v_proj": {
1737
+ "group_size": 64,
1738
+ "bits": 8
1739
+ },
1740
+ "model.layers.24.self_attn.o_proj": {
1741
+ "group_size": 64,
1742
+ "bits": 8
1743
+ },
1744
+ "model.layers.25.self_attn.q_proj": {
1745
+ "group_size": 64,
1746
+ "bits": 8
1747
+ },
1748
+ "model.layers.25.self_attn.k_proj": {
1749
+ "group_size": 64,
1750
+ "bits": 8
1751
+ },
1752
+ "model.layers.25.self_attn.v_proj": {
1753
+ "group_size": 64,
1754
+ "bits": 8
1755
+ },
1756
+ "model.layers.25.self_attn.o_proj": {
1757
+ "group_size": 64,
1758
+ "bits": 8
1759
+ },
1760
+ "model.layers.26.self_attn.q_proj": {
1761
+ "group_size": 64,
1762
+ "bits": 8
1763
+ },
1764
+ "model.layers.26.self_attn.k_proj": {
1765
+ "group_size": 64,
1766
+ "bits": 8
1767
+ },
1768
+ "model.layers.26.self_attn.v_proj": {
1769
+ "group_size": 64,
1770
+ "bits": 8
1771
+ },
1772
+ "model.layers.26.self_attn.o_proj": {
1773
+ "group_size": 64,
1774
+ "bits": 8
1775
+ },
1776
+ "model.layers.27.self_attn.q_proj": {
1777
+ "group_size": 64,
1778
+ "bits": 8
1779
+ },
1780
+ "model.layers.27.self_attn.k_proj": {
1781
+ "group_size": 64,
1782
+ "bits": 8
1783
+ },
1784
+ "model.layers.27.self_attn.v_proj": {
1785
+ "group_size": 64,
1786
+ "bits": 8
1787
+ },
1788
+ "model.layers.27.self_attn.o_proj": {
1789
+ "group_size": 64,
1790
+ "bits": 8
1791
+ },
1792
+ "model.layers.28.self_attn.q_proj": {
1793
+ "group_size": 64,
1794
+ "bits": 8
1795
+ },
1796
+ "model.layers.28.self_attn.k_proj": {
1797
+ "group_size": 64,
1798
+ "bits": 8
1799
+ },
1800
+ "model.layers.28.self_attn.v_proj": {
1801
+ "group_size": 64,
1802
+ "bits": 8
1803
+ },
1804
+ "model.layers.28.self_attn.o_proj": {
1805
+ "group_size": 64,
1806
+ "bits": 8
1807
+ },
1808
+ "model.layers.29.self_attn.q_proj": {
1809
+ "group_size": 64,
1810
+ "bits": 8
1811
+ },
1812
+ "model.layers.29.self_attn.k_proj": {
1813
+ "group_size": 64,
1814
+ "bits": 8
1815
+ },
1816
+ "model.layers.29.self_attn.v_proj": {
1817
+ "group_size": 64,
1818
+ "bits": 8
1819
+ },
1820
+ "model.layers.29.self_attn.o_proj": {
1821
+ "group_size": 64,
1822
+ "bits": 8
1823
+ },
1824
+ "model.layers.30.self_attn.q_proj": {
1825
+ "group_size": 64,
1826
+ "bits": 8
1827
+ },
1828
+ "model.layers.30.self_attn.k_proj": {
1829
+ "group_size": 64,
1830
+ "bits": 8
1831
+ },
1832
+ "model.layers.30.self_attn.v_proj": {
1833
+ "group_size": 64,
1834
+ "bits": 8
1835
+ },
1836
+ "model.layers.30.self_attn.o_proj": {
1837
+ "group_size": 64,
1838
+ "bits": 8
1839
+ },
1840
+ "model.layers.31.self_attn.q_proj": {
1841
+ "group_size": 64,
1842
+ "bits": 8
1843
+ },
1844
+ "model.layers.31.self_attn.k_proj": {
1845
+ "group_size": 64,
1846
+ "bits": 8
1847
+ },
1848
+ "model.layers.31.self_attn.v_proj": {
1849
+ "group_size": 64,
1850
+ "bits": 8
1851
+ },
1852
+ "model.layers.31.self_attn.o_proj": {
1853
+ "group_size": 64,
1854
+ "bits": 8
1855
+ },
1856
+ "model.layers.32.self_attn.q_proj": {
1857
+ "group_size": 64,
1858
+ "bits": 8
1859
+ },
1860
+ "model.layers.32.self_attn.k_proj": {
1861
+ "group_size": 64,
1862
+ "bits": 8
1863
+ },
1864
+ "model.layers.32.self_attn.v_proj": {
1865
+ "group_size": 64,
1866
+ "bits": 8
1867
+ },
1868
+ "model.layers.32.self_attn.o_proj": {
1869
+ "group_size": 64,
1870
+ "bits": 8
1871
+ },
1872
+ "model.layers.33.self_attn.q_proj": {
1873
+ "group_size": 64,
1874
+ "bits": 8
1875
+ },
1876
+ "model.layers.33.self_attn.k_proj": {
1877
+ "group_size": 64,
1878
+ "bits": 8
1879
+ },
1880
+ "model.layers.33.self_attn.v_proj": {
1881
+ "group_size": 64,
1882
+ "bits": 8
1883
+ },
1884
+ "model.layers.33.self_attn.o_proj": {
1885
+ "group_size": 64,
1886
+ "bits": 8
1887
+ },
1888
+ "model.layers.34.self_attn.q_proj": {
1889
+ "group_size": 64,
1890
+ "bits": 8
1891
+ },
1892
+ "model.layers.34.self_attn.k_proj": {
1893
+ "group_size": 64,
1894
+ "bits": 8
1895
+ },
1896
+ "model.layers.34.self_attn.v_proj": {
1897
+ "group_size": 64,
1898
+ "bits": 8
1899
+ },
1900
+ "model.layers.34.self_attn.o_proj": {
1901
+ "group_size": 64,
1902
+ "bits": 8
1903
+ },
1904
+ "model.layers.35.self_attn.q_proj": {
1905
+ "group_size": 64,
1906
+ "bits": 8
1907
+ },
1908
+ "model.layers.35.self_attn.k_proj": {
1909
+ "group_size": 64,
1910
+ "bits": 8
1911
+ },
1912
+ "model.layers.35.self_attn.v_proj": {
1913
+ "group_size": 64,
1914
+ "bits": 8
1915
+ },
1916
+ "model.layers.35.self_attn.o_proj": {
1917
+ "group_size": 64,
1918
+ "bits": 8
1919
+ },
1920
+ "model.layers.36.self_attn.q_proj": {
1921
+ "group_size": 64,
1922
+ "bits": 8
1923
+ },
1924
+ "model.layers.36.self_attn.k_proj": {
1925
+ "group_size": 64,
1926
+ "bits": 8
1927
+ },
1928
+ "model.layers.36.self_attn.v_proj": {
1929
+ "group_size": 64,
1930
+ "bits": 8
1931
+ },
1932
+ "model.layers.36.self_attn.o_proj": {
1933
+ "group_size": 64,
1934
+ "bits": 8
1935
+ },
1936
+ "model.layers.37.self_attn.q_proj": {
1937
+ "group_size": 64,
1938
+ "bits": 8
1939
+ },
1940
+ "model.layers.37.self_attn.k_proj": {
1941
+ "group_size": 64,
1942
+ "bits": 8
1943
+ },
1944
+ "model.layers.37.self_attn.v_proj": {
1945
+ "group_size": 64,
1946
+ "bits": 8
1947
+ },
1948
+ "model.layers.37.self_attn.o_proj": {
1949
+ "group_size": 64,
1950
+ "bits": 8
1951
+ },
1952
+ "model.layers.38.self_attn.q_proj": {
1953
+ "group_size": 64,
1954
+ "bits": 8
1955
+ },
1956
+ "model.layers.38.self_attn.k_proj": {
1957
+ "group_size": 64,
1958
+ "bits": 8
1959
+ },
1960
+ "model.layers.38.self_attn.v_proj": {
1961
+ "group_size": 64,
1962
+ "bits": 8
1963
+ },
1964
+ "model.layers.38.self_attn.o_proj": {
1965
+ "group_size": 64,
1966
+ "bits": 8
1967
+ },
1968
+ "model.layers.39.self_attn.q_proj": {
1969
+ "group_size": 64,
1970
+ "bits": 8
1971
+ },
1972
+ "model.layers.39.self_attn.k_proj": {
1973
+ "group_size": 64,
1974
+ "bits": 8
1975
+ },
1976
+ "model.layers.39.self_attn.v_proj": {
1977
+ "group_size": 64,
1978
+ "bits": 8
1979
+ },
1980
+ "model.layers.39.self_attn.o_proj": {
1981
+ "group_size": 64,
1982
+ "bits": 8
1983
+ },
1984
+ "model.layers.40.self_attn.q_proj": {
1985
+ "group_size": 64,
1986
+ "bits": 8
1987
+ },
1988
+ "model.layers.40.self_attn.k_proj": {
1989
+ "group_size": 64,
1990
+ "bits": 8
1991
+ },
1992
+ "model.layers.40.self_attn.v_proj": {
1993
+ "group_size": 64,
1994
+ "bits": 8
1995
+ },
1996
+ "model.layers.40.self_attn.o_proj": {
1997
+ "group_size": 64,
1998
+ "bits": 8
1999
+ },
2000
+ "model.layers.41.self_attn.q_proj": {
2001
+ "group_size": 64,
2002
+ "bits": 8
2003
+ },
2004
+ "model.layers.41.self_attn.k_proj": {
2005
+ "group_size": 64,
2006
+ "bits": 8
2007
+ },
2008
+ "model.layers.41.self_attn.v_proj": {
2009
+ "group_size": 64,
2010
+ "bits": 8
2011
+ },
2012
+ "model.layers.41.self_attn.o_proj": {
2013
+ "group_size": 64,
2014
+ "bits": 8
2015
+ },
2016
+ "model.layers.42.self_attn.q_proj": {
2017
+ "group_size": 64,
2018
+ "bits": 8
2019
+ },
2020
+ "model.layers.42.self_attn.k_proj": {
2021
+ "group_size": 64,
2022
+ "bits": 8
2023
+ },
2024
+ "model.layers.42.self_attn.v_proj": {
2025
+ "group_size": 64,
2026
+ "bits": 8
2027
+ },
2028
+ "model.layers.42.self_attn.o_proj": {
2029
+ "group_size": 64,
2030
+ "bits": 8
2031
+ },
2032
+ "model.layers.43.self_attn.q_proj": {
2033
+ "group_size": 64,
2034
+ "bits": 8
2035
+ },
2036
+ "model.layers.43.self_attn.k_proj": {
2037
+ "group_size": 64,
2038
+ "bits": 8
2039
+ },
2040
+ "model.layers.43.self_attn.v_proj": {
2041
+ "group_size": 64,
2042
+ "bits": 8
2043
+ },
2044
+ "model.layers.43.self_attn.o_proj": {
2045
+ "group_size": 64,
2046
+ "bits": 8
2047
+ },
2048
+ "model.layers.44.self_attn.q_proj": {
2049
+ "group_size": 64,
2050
+ "bits": 8
2051
+ },
2052
+ "model.layers.44.self_attn.k_proj": {
2053
+ "group_size": 64,
2054
+ "bits": 8
2055
+ },
2056
+ "model.layers.44.self_attn.v_proj": {
2057
+ "group_size": 64,
2058
+ "bits": 8
2059
+ },
2060
+ "model.layers.44.self_attn.o_proj": {
2061
+ "group_size": 64,
2062
+ "bits": 8
2063
+ },
2064
+ "model.layers.45.self_attn.q_proj": {
2065
+ "group_size": 64,
2066
+ "bits": 8
2067
+ },
2068
+ "model.layers.45.self_attn.k_proj": {
2069
+ "group_size": 64,
2070
+ "bits": 8
2071
+ },
2072
+ "model.layers.45.self_attn.v_proj": {
2073
+ "group_size": 64,
2074
+ "bits": 8
2075
+ },
2076
+ "model.layers.45.self_attn.o_proj": {
2077
+ "group_size": 64,
2078
+ "bits": 8
2079
+ },
2080
+ "model.layers.46.self_attn.q_proj": {
2081
+ "group_size": 64,
2082
+ "bits": 8
2083
+ },
2084
+ "model.layers.46.self_attn.k_proj": {
2085
+ "group_size": 64,
2086
+ "bits": 8
2087
+ },
2088
+ "model.layers.46.self_attn.v_proj": {
2089
+ "group_size": 64,
2090
+ "bits": 8
2091
+ },
2092
+ "model.layers.46.self_attn.o_proj": {
2093
+ "group_size": 64,
2094
+ "bits": 8
2095
+ },
2096
+ "model.layers.47.self_attn.q_proj": {
2097
+ "group_size": 64,
2098
+ "bits": 8
2099
+ },
2100
+ "model.layers.47.self_attn.k_proj": {
2101
+ "group_size": 64,
2102
+ "bits": 8
2103
+ },
2104
+ "model.layers.47.self_attn.v_proj": {
2105
+ "group_size": 64,
2106
+ "bits": 8
2107
+ },
2108
+ "model.layers.47.self_attn.o_proj": {
2109
+ "group_size": 64,
2110
+ "bits": 8
2111
+ },
2112
+ "model.layers.48.self_attn.q_proj": {
2113
+ "group_size": 64,
2114
+ "bits": 8
2115
+ },
2116
+ "model.layers.48.self_attn.k_proj": {
2117
+ "group_size": 64,
2118
+ "bits": 8
2119
+ },
2120
+ "model.layers.48.self_attn.v_proj": {
2121
+ "group_size": 64,
2122
+ "bits": 8
2123
+ },
2124
+ "model.layers.48.self_attn.o_proj": {
2125
+ "group_size": 64,
2126
+ "bits": 8
2127
+ },
2128
+ "model.layers.49.self_attn.q_proj": {
2129
+ "group_size": 64,
2130
+ "bits": 8
2131
+ },
2132
+ "model.layers.49.self_attn.k_proj": {
2133
+ "group_size": 64,
2134
+ "bits": 8
2135
+ },
2136
+ "model.layers.49.self_attn.v_proj": {
2137
+ "group_size": 64,
2138
+ "bits": 8
2139
+ },
2140
+ "model.layers.49.self_attn.o_proj": {
2141
+ "group_size": 64,
2142
+ "bits": 8
2143
+ },
2144
+ "model.layers.50.self_attn.q_proj": {
2145
+ "group_size": 64,
2146
+ "bits": 8
2147
+ },
2148
+ "model.layers.50.self_attn.k_proj": {
2149
+ "group_size": 64,
2150
+ "bits": 8
2151
+ },
2152
+ "model.layers.50.self_attn.v_proj": {
2153
+ "group_size": 64,
2154
+ "bits": 8
2155
+ },
2156
+ "model.layers.50.self_attn.o_proj": {
2157
+ "group_size": 64,
2158
+ "bits": 8
2159
+ },
2160
+ "model.layers.51.self_attn.q_proj": {
2161
+ "group_size": 64,
2162
+ "bits": 8
2163
+ },
2164
+ "model.layers.51.self_attn.k_proj": {
2165
+ "group_size": 64,
2166
+ "bits": 8
2167
+ },
2168
+ "model.layers.51.self_attn.v_proj": {
2169
+ "group_size": 64,
2170
+ "bits": 8
2171
+ },
2172
+ "model.layers.51.self_attn.o_proj": {
2173
+ "group_size": 64,
2174
+ "bits": 8
2175
+ },
2176
+ "model.layers.52.self_attn.q_proj": {
2177
+ "group_size": 64,
2178
+ "bits": 8
2179
+ },
2180
+ "model.layers.52.self_attn.k_proj": {
2181
+ "group_size": 64,
2182
+ "bits": 8
2183
+ },
2184
+ "model.layers.52.self_attn.v_proj": {
2185
+ "group_size": 64,
2186
+ "bits": 8
2187
+ },
2188
+ "model.layers.52.self_attn.o_proj": {
2189
+ "group_size": 64,
2190
+ "bits": 8
2191
+ },
2192
+ "model.layers.53.self_attn.q_proj": {
2193
+ "group_size": 64,
2194
+ "bits": 8
2195
+ },
2196
+ "model.layers.53.self_attn.k_proj": {
2197
+ "group_size": 64,
2198
+ "bits": 8
2199
+ },
2200
+ "model.layers.53.self_attn.v_proj": {
2201
+ "group_size": 64,
2202
+ "bits": 8
2203
+ },
2204
+ "model.layers.53.self_attn.o_proj": {
2205
+ "group_size": 64,
2206
+ "bits": 8
2207
+ },
2208
+ "model.layers.54.self_attn.q_proj": {
2209
+ "group_size": 64,
2210
+ "bits": 8
2211
+ },
2212
+ "model.layers.54.self_attn.k_proj": {
2213
+ "group_size": 64,
2214
+ "bits": 8
2215
+ },
2216
+ "model.layers.54.self_attn.v_proj": {
2217
+ "group_size": 64,
2218
+ "bits": 8
2219
+ },
2220
+ "model.layers.54.self_attn.o_proj": {
2221
+ "group_size": 64,
2222
+ "bits": 8
2223
+ },
2224
+ "model.layers.55.self_attn.q_proj": {
2225
+ "group_size": 64,
2226
+ "bits": 8
2227
+ },
2228
+ "model.layers.55.self_attn.k_proj": {
2229
+ "group_size": 64,
2230
+ "bits": 8
2231
+ },
2232
+ "model.layers.55.self_attn.v_proj": {
2233
+ "group_size": 64,
2234
+ "bits": 8
2235
+ },
2236
+ "model.layers.55.self_attn.o_proj": {
2237
+ "group_size": 64,
2238
+ "bits": 8
2239
+ },
2240
+ "model.layers.56.self_attn.q_proj": {
2241
+ "group_size": 64,
2242
+ "bits": 8
2243
+ },
2244
+ "model.layers.56.self_attn.k_proj": {
2245
+ "group_size": 64,
2246
+ "bits": 8
2247
+ },
2248
+ "model.layers.56.self_attn.v_proj": {
2249
+ "group_size": 64,
2250
+ "bits": 8
2251
+ },
2252
+ "model.layers.56.self_attn.o_proj": {
2253
+ "group_size": 64,
2254
+ "bits": 8
2255
+ },
2256
+ "model.layers.57.self_attn.q_proj": {
2257
+ "group_size": 64,
2258
+ "bits": 8
2259
+ },
2260
+ "model.layers.57.self_attn.k_proj": {
2261
+ "group_size": 64,
2262
+ "bits": 8
2263
+ },
2264
+ "model.layers.57.self_attn.v_proj": {
2265
+ "group_size": 64,
2266
+ "bits": 8
2267
+ },
2268
+ "model.layers.57.self_attn.o_proj": {
2269
+ "group_size": 64,
2270
+ "bits": 8
2271
+ },
2272
+ "model.layers.58.self_attn.q_proj": {
2273
+ "group_size": 64,
2274
+ "bits": 8
2275
+ },
2276
+ "model.layers.58.self_attn.k_proj": {
2277
+ "group_size": 64,
2278
+ "bits": 8
2279
+ },
2280
+ "model.layers.58.self_attn.v_proj": {
2281
+ "group_size": 64,
2282
+ "bits": 8
2283
+ },
2284
+ "model.layers.58.self_attn.o_proj": {
2285
+ "group_size": 64,
2286
+ "bits": 8
2287
+ },
2288
+ "model.layers.59.self_attn.q_proj": {
2289
+ "group_size": 64,
2290
+ "bits": 8
2291
+ },
2292
+ "model.layers.59.self_attn.k_proj": {
2293
+ "group_size": 64,
2294
+ "bits": 8
2295
+ },
2296
+ "model.layers.59.self_attn.v_proj": {
2297
+ "group_size": 64,
2298
+ "bits": 8
2299
+ },
2300
+ "model.layers.59.self_attn.o_proj": {
2301
+ "group_size": 64,
2302
+ "bits": 8
2303
+ },
2304
+ "model.layers.60.self_attn.q_proj": {
2305
+ "group_size": 64,
2306
+ "bits": 8
2307
+ },
2308
+ "model.layers.60.self_attn.k_proj": {
2309
+ "group_size": 64,
2310
+ "bits": 8
2311
+ },
2312
+ "model.layers.60.self_attn.v_proj": {
2313
+ "group_size": 64,
2314
+ "bits": 8
2315
+ },
2316
+ "model.layers.60.self_attn.o_proj": {
2317
+ "group_size": 64,
2318
+ "bits": 8
2319
+ },
2320
+ "model.layers.61.self_attn.q_proj": {
2321
+ "group_size": 64,
2322
+ "bits": 8
2323
+ },
2324
+ "model.layers.61.self_attn.k_proj": {
2325
+ "group_size": 64,
2326
+ "bits": 8
2327
+ },
2328
+ "model.layers.61.self_attn.v_proj": {
2329
+ "group_size": 64,
2330
+ "bits": 8
2331
+ },
2332
+ "model.layers.61.self_attn.o_proj": {
2333
+ "group_size": 64,
2334
+ "bits": 8
2335
+ },
2336
+ "model.layers.62.self_attn.q_proj": {
2337
+ "group_size": 64,
2338
+ "bits": 8
2339
+ },
2340
+ "model.layers.62.self_attn.k_proj": {
2341
+ "group_size": 64,
2342
+ "bits": 8
2343
+ },
2344
+ "model.layers.62.self_attn.v_proj": {
2345
+ "group_size": 64,
2346
+ "bits": 8
2347
+ },
2348
+ "model.layers.62.self_attn.o_proj": {
2349
+ "group_size": 64,
2350
+ "bits": 8
2351
+ },
2352
+ "model.layers.63.self_attn.q_proj": {
2353
+ "group_size": 64,
2354
+ "bits": 8
2355
+ },
2356
+ "model.layers.63.self_attn.k_proj": {
2357
+ "group_size": 64,
2358
+ "bits": 8
2359
+ },
2360
+ "model.layers.63.self_attn.v_proj": {
2361
+ "group_size": 64,
2362
+ "bits": 8
2363
+ },
2364
+ "model.layers.63.self_attn.o_proj": {
2365
+ "group_size": 64,
2366
+ "bits": 8
2367
+ },
2368
+ "model.layers.64.self_attn.q_proj": {
2369
+ "group_size": 64,
2370
+ "bits": 8
2371
+ },
2372
+ "model.layers.64.self_attn.k_proj": {
2373
+ "group_size": 64,
2374
+ "bits": 8
2375
+ },
2376
+ "model.layers.64.self_attn.v_proj": {
2377
+ "group_size": 64,
2378
+ "bits": 8
2379
+ },
2380
+ "model.layers.64.self_attn.o_proj": {
2381
+ "group_size": 64,
2382
+ "bits": 8
2383
+ },
2384
+ "model.layers.65.self_attn.q_proj": {
2385
+ "group_size": 64,
2386
+ "bits": 8
2387
+ },
2388
+ "model.layers.65.self_attn.k_proj": {
2389
+ "group_size": 64,
2390
+ "bits": 8
2391
+ },
2392
+ "model.layers.65.self_attn.v_proj": {
2393
+ "group_size": 64,
2394
+ "bits": 8
2395
+ },
2396
+ "model.layers.65.self_attn.o_proj": {
2397
+ "group_size": 64,
2398
+ "bits": 8
2399
+ },
2400
+ "model.layers.66.self_attn.q_proj": {
2401
+ "group_size": 64,
2402
+ "bits": 8
2403
+ },
2404
+ "model.layers.66.self_attn.k_proj": {
2405
+ "group_size": 64,
2406
+ "bits": 8
2407
+ },
2408
+ "model.layers.66.self_attn.v_proj": {
2409
+ "group_size": 64,
2410
+ "bits": 8
2411
+ },
2412
+ "model.layers.66.self_attn.o_proj": {
2413
+ "group_size": 64,
2414
+ "bits": 8
2415
+ },
2416
+ "model.layers.67.self_attn.q_proj": {
2417
+ "group_size": 64,
2418
+ "bits": 8
2419
+ },
2420
+ "model.layers.67.self_attn.k_proj": {
2421
+ "group_size": 64,
2422
+ "bits": 8
2423
+ },
2424
+ "model.layers.67.self_attn.v_proj": {
2425
+ "group_size": 64,
2426
+ "bits": 8
2427
+ },
2428
+ "model.layers.67.self_attn.o_proj": {
2429
+ "group_size": 64,
2430
+ "bits": 8
2431
+ },
2432
+ "model.layers.68.self_attn.q_proj": {
2433
+ "group_size": 64,
2434
+ "bits": 8
2435
+ },
2436
+ "model.layers.68.self_attn.k_proj": {
2437
+ "group_size": 64,
2438
+ "bits": 8
2439
+ },
2440
+ "model.layers.68.self_attn.v_proj": {
2441
+ "group_size": 64,
2442
+ "bits": 8
2443
+ },
2444
+ "model.layers.68.self_attn.o_proj": {
2445
+ "group_size": 64,
2446
+ "bits": 8
2447
+ },
2448
+ "model.layers.69.self_attn.q_proj": {
2449
+ "group_size": 64,
2450
+ "bits": 8
2451
+ },
2452
+ "model.layers.69.self_attn.k_proj": {
2453
+ "group_size": 64,
2454
+ "bits": 8
2455
+ },
2456
+ "model.layers.69.self_attn.v_proj": {
2457
+ "group_size": 64,
2458
+ "bits": 8
2459
+ },
2460
+ "model.layers.69.self_attn.o_proj": {
2461
+ "group_size": 64,
2462
+ "bits": 8
2463
+ },
2464
+ "lm_head": {
2465
+ "group_size": 32,
2466
+ "bits": 8
2467
+ }
2468
+ },
2469
+ "rope_theta": 10000000,
2470
+ "routed_scaling_factor": null,
2471
+ "scoring_func": "sigmoid",
2472
+ "sliding_window": 128,
2473
+ "sliding_window_size": 128,
2474
+ "swa_head_dim": 192,
2475
+ "swa_num_attention_heads": 128,
2476
+ "swa_num_key_value_heads": 8,
2477
+ "swa_rope_theta": 10000,
2478
+ "swa_v_head_dim": 128,
2479
+ "tie_word_embeddings": false,
2480
+ "topk_group": 1,
2481
+ "topk_method": "noaux_tc",
2482
+ "torch_dtype": "bfloat16",
2483
+ "transformers_version": "4.57.1",
2484
+ "use_cache": true,
2485
+ "v_head_dim": 128,
2486
+ "vocab_size": 152576
2487
+ }
configuration_mimo_v2.py ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ #
3
+ # Copyright 2026 Xiaomi Corporation.
4
+ # Copyright 2026 The HuggingFace Inc. team.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+
18
+ from transformers.configuration_utils import PretrainedConfig
19
+ from transformers.modeling_rope_utils import rope_config_validation
20
+ from transformers.utils import logging
21
+
22
+
23
+ logger = logging.get_logger(__name__)
24
+
25
+
26
+ _MIMOV2_ATTENTION_PROJECTION_LAYOUTS = {"split", "fused_qkv"}
27
+
28
+ _MIMOV2_SPLIT_TP_PLAN = {
29
+ "layers.*.self_attn.q_proj": "colwise",
30
+ "layers.*.self_attn.k_proj": "colwise",
31
+ "layers.*.self_attn.v_proj": "colwise",
32
+ "layers.*.self_attn.o_proj": "rowwise",
33
+ "layers.*.mlp.gate_proj": "colwise",
34
+ "layers.*.mlp.up_proj": "colwise",
35
+ "layers.*.mlp.down_proj": "rowwise",
36
+ }
37
+
38
+ _MIMOV2_FUSED_QKV_TP_PLAN = {
39
+ "layers.*.self_attn.qkv_proj": "colwise",
40
+ "layers.*.self_attn.o_proj": "rowwise",
41
+ "layers.*.mlp.gate_proj": "colwise",
42
+ "layers.*.mlp.up_proj": "colwise",
43
+ "layers.*.mlp.down_proj": "rowwise",
44
+ }
45
+
46
+ _MIMOV2_PP_PLAN = {
47
+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),
48
+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
49
+ "norm": (["hidden_states"], ["hidden_states"]),
50
+ }
51
+
52
+
53
+ class MiMoV2Config(PretrainedConfig):
54
+
55
+ model_type = "mimo_v2"
56
+ keys_to_ignore_at_inference = ["past_key_values"]
57
+
58
+ base_model_tp_plan = _MIMOV2_SPLIT_TP_PLAN
59
+ base_model_pp_plan = _MIMOV2_PP_PLAN
60
+
61
+ attribute_map = {
62
+ "num_local_experts": "n_routed_experts",
63
+ }
64
+
65
+ def __init__(
66
+ self,
67
+ vocab_size=151936,
68
+ hidden_size=4096,
69
+ intermediate_size=22016,
70
+ num_hidden_layers=32,
71
+ num_attention_heads=32,
72
+ num_key_value_heads=32,
73
+ hidden_act="silu",
74
+ max_position_embeddings=32768,
75
+ initializer_range=0.02,
76
+ layernorm_epsilon=1e-6,
77
+ use_cache=True,
78
+ tie_word_embeddings=False,
79
+ rope_theta=10000.0,
80
+ rope_scaling=None,
81
+ attention_dropout=0.0,
82
+ attention_bias=False,
83
+ attention_value_scale=None,
84
+ head_dim=None,
85
+ v_head_dim=None,
86
+ swa_num_attention_heads=None,
87
+ swa_num_key_value_heads=None,
88
+ swa_head_dim=None,
89
+ swa_v_head_dim=None,
90
+ swa_rope_theta=None,
91
+ sliding_window=None,
92
+ sliding_window_size=None,
93
+ add_full_attention_sink_bias=False,
94
+ add_swa_attention_sink_bias=False,
95
+ hybrid_block_size=None,
96
+ hybrid_layer_pattern=None,
97
+ partial_rotary_factor=1.0,
98
+ n_routed_experts=None,
99
+ moe_intermediate_size=None,
100
+ num_experts_per_tok=None,
101
+ routed_scaling_factor=None,
102
+ scoring_func="sigmoid",
103
+ topk_method="noaux_tc",
104
+ n_group=None,
105
+ topk_group=None,
106
+ norm_topk_prob=True,
107
+ moe_layer_freq=None,
108
+ attention_projection_layout="split",
109
+ **kwargs,
110
+ ):
111
+ rope_parameters = kwargs.pop("rope_parameters", None)
112
+ if rope_scaling is None and rope_parameters is not None:
113
+ rope_scaling = rope_parameters
114
+
115
+ if attention_projection_layout is None:
116
+ attention_projection_layout = "split"
117
+ if attention_projection_layout not in _MIMOV2_ATTENTION_PROJECTION_LAYOUTS:
118
+ raise ValueError(f"Unsupported MiMoV2 attention projection layout: {attention_projection_layout}")
119
+
120
+ self.attention_projection_layout = attention_projection_layout
121
+ self.base_model_tp_plan = (
122
+ _MIMOV2_FUSED_QKV_TP_PLAN.copy()
123
+ if attention_projection_layout == "fused_qkv"
124
+ else _MIMOV2_SPLIT_TP_PLAN.copy()
125
+ )
126
+ self.base_model_pp_plan = _MIMOV2_PP_PLAN.copy()
127
+
128
+ self.vocab_size = vocab_size
129
+ self.max_position_embeddings = max_position_embeddings
130
+ self.hidden_size = hidden_size
131
+ self.intermediate_size = intermediate_size
132
+ self.num_hidden_layers = num_hidden_layers
133
+ self.num_attention_heads = num_attention_heads
134
+
135
+ if num_key_value_heads is None:
136
+ num_key_value_heads = num_attention_heads
137
+ if num_attention_heads % num_key_value_heads != 0:
138
+ raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
139
+
140
+ self.num_key_value_heads = num_key_value_heads
141
+ self.hidden_act = hidden_act
142
+ self.initializer_range = initializer_range
143
+ self.layernorm_epsilon = layernorm_epsilon
144
+ self.use_cache = use_cache
145
+ self.rope_theta = rope_theta
146
+ self.rope_scaling = rope_scaling
147
+ self.attention_dropout = attention_dropout
148
+ self.attention_bias = attention_bias
149
+ self.attention_value_scale = attention_value_scale
150
+
151
+ self.head_dim = head_dim if head_dim is not None else hidden_size // num_attention_heads
152
+ self.v_head_dim = v_head_dim if v_head_dim is not None else self.head_dim
153
+ self.swa_num_attention_heads = (
154
+ swa_num_attention_heads if swa_num_attention_heads is not None else num_attention_heads
155
+ )
156
+ self.swa_num_key_value_heads = (
157
+ swa_num_key_value_heads if swa_num_key_value_heads is not None else num_key_value_heads
158
+ )
159
+ if self.swa_num_attention_heads % self.swa_num_key_value_heads != 0:
160
+ raise ValueError("swa_num_attention_heads must be divisible by swa_num_key_value_heads")
161
+ self.swa_head_dim = swa_head_dim if swa_head_dim is not None else self.head_dim
162
+ self.swa_v_head_dim = swa_v_head_dim if swa_v_head_dim is not None else self.swa_head_dim
163
+ self.swa_rope_theta = swa_rope_theta if swa_rope_theta is not None else rope_theta
164
+
165
+ if sliding_window is None:
166
+ sliding_window = sliding_window_size
167
+ self.sliding_window = sliding_window
168
+ self.sliding_window_size = sliding_window_size if sliding_window_size is not None else sliding_window
169
+ self.add_full_attention_sink_bias = add_full_attention_sink_bias
170
+ self.add_swa_attention_sink_bias = add_swa_attention_sink_bias
171
+
172
+ if hybrid_block_size is not None and hybrid_layer_pattern is None:
173
+ hybrid_layer_pattern = [0 if ((i + 1) % hybrid_block_size == 0) else 1 for i in range(num_hidden_layers)]
174
+ elif hybrid_layer_pattern is None:
175
+ hybrid_layer_pattern = [0] * num_hidden_layers
176
+ if len(hybrid_layer_pattern) != num_hidden_layers:
177
+ raise ValueError("hybrid_layer_pattern length must match num_hidden_layers")
178
+ self.hybrid_block_size = hybrid_block_size
179
+ self.hybrid_layer_pattern = hybrid_layer_pattern
180
+
181
+ self.partial_rotary_factor = partial_rotary_factor
182
+
183
+ self.n_routed_experts = n_routed_experts
184
+ self.moe_intermediate_size = moe_intermediate_size if moe_intermediate_size is not None else intermediate_size
185
+ self.num_experts_per_tok = num_experts_per_tok
186
+ self.routed_scaling_factor = routed_scaling_factor
187
+ self.scoring_func = scoring_func
188
+ self.topk_method = topk_method
189
+ self.n_group = n_group
190
+ self.topk_group = topk_group
191
+ self.norm_topk_prob = norm_topk_prob
192
+ if isinstance(moe_layer_freq, int):
193
+ moe_layer_freq = [moe_layer_freq > 0 and i % moe_layer_freq == 0 for i in range(num_hidden_layers)]
194
+ elif moe_layer_freq is None:
195
+ moe_layer_freq = [False] * num_hidden_layers
196
+ if len(moe_layer_freq) != num_hidden_layers:
197
+ raise ValueError("moe_layer_freq length must match num_hidden_layers")
198
+ self.moe_layer_freq = moe_layer_freq
199
+
200
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
201
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
202
+ rope_config_validation(self)
203
+
204
+ super().__init__(
205
+ tie_word_embeddings=tie_word_embeddings,
206
+ **kwargs,
207
+ )
208
+
209
+ __all__ = ["MiMoV2Config"]
model-00002-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30892b06a1f3cb2771d8b0a52c817ad67bc20fd1c954328d4634c0c66555f6b3
3
+ size 3925868954
model-00004-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b82e6fba532dd19e900adb8910602144a8bac6a4350d0f650aa2e106e0e80c22
3
+ size 4214714387
model-00007-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbc57780a1736eb561251d47fedb1c90afc74a6fc9cc882b0601b3abead80b2c
3
+ size 4214714399
model-00008-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ede0a041e355a8894d3d951d99be5cf27690544df8be9dcbc2a8861e9bbdcde9
3
+ size 3925868954
model-00030-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c12532574a45b96189a526d6feb9cf31f59e3a93aae37326c00d74dc44f88bba
3
+ size 3925868951
model-00035-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a2d1e43bd324a94637a094f1db23b94f4d55f3fae122f5d1168e04d46f169eb5
3
+ size 3925868957
model-00036-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fa6ef7b04c3e06d768dc1a4c899030a167518d69a9ebb422c17bbbf273dba58
3
+ size 3925868951
model-00039-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d446004eef21fa755981191c2fe9565bf94fb4a8f5b8e81e51b3c729890ba06b
3
+ size 3925868951
model-00040-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9a632d8cdfe3ace1cfca33b2986dd21c98a6c379d8587c79e5f05c4a2d03426
3
+ size 4214714421
model-00043-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a42c0605c0885daa40a3cbcc2e7a82340a6f9ad1e63f472b8b3e701931d8d07
3
+ size 4214714071
model-00045-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ad2bfb67057c7dadeae29c0c833795fd04259edf4edd275c66e5eee383c20f0
3
+ size 3925868951
model-00046-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:247db619425d47a9c09deb5b2dce8a06eb16dbb5855b590589c5c0e2053ecf85
3
+ size 4214714411
model-00049-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:323ab3e2353a0193d33cce9e993351f758c90e0004c5ae38399aa5ab16902371
3
+ size 4214714407
model-00071-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9686b0ee7bbec64be90c911012e4f8daf669dd60477d42f720191f8bffb256fd
3
+ size 3925868957
model-00074-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe58d9c0157dc51554dc6248614ca6798425622d98be7246ce3634cab6001baa
3
+ size 3925868957
model-00078-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:077febb52b4981a41cf9fc13efbdea3bad693125d062764e04e3bc3f62df4fde
3
+ size 3925868951
model-00082-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a92ffd4dea91a5d25d62d360496a8ce2e34239db1fba8fc5e14ba996eefd472
3
+ size 4214714413
model-00087-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da9254bac4f5aa28ff8aeb16f4d659cc95b5c13029792bc67170fce4ee550fb7
3
+ size 3925868951
model-00088-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38766af4ba77571bad50ab3523c6f0083f1a2951be7f0e50343799dc5a277c59
3
+ size 4214714425
model-00111-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59eb92b43066da868f22a1ea70a13de5c6828c6116977e64fd1e17708b8eb6b9
3
+ size 3925868951
model-00114-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea5e0bd0954ce96d055c3f16426d15d643054fcbc9049c4a20a149c6a7346449
3
+ size 3925868951
model-00117-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb1b3a91f91e48474a003814ac40989c8f647baf715af5d4d35fb290a4ad9f04
3
+ size 3925868951
model-00118-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebb3c300359531f755409a16c7e2e7c79dbb01cab0c5240e7bf7874bf0d482cf
3
+ size 4214714405
model-00120-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09717a52db707715f2b700f706721f56536e1dbe82623ce5acd5f440d2c500c7
3
+ size 3925868951
model-00123-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1cba6c39e1516a2cd345bcf63174bca2199ca1dbc0c9ff118b447e17ee47d415
3
+ size 3925868951
model-00125-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30b16a29199bf027316595682e0d19929c8bc1443b7641da3a670c015c4b7dce
3
+ size 3925868957
model-00126-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8773dcc917e8e0084111a826263c523bb1fdcf452a43948125e0267d7f351b0
3
+ size 3925868951
model-00129-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca340b074e08c73a7cc7cd92ef62c0b60f056602f2df02dbcf615351d3f7ecc5
3
+ size 3925868951
model-00142-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1ae3c3ea2e5f668cc1f6ddf24ec9dd3f2331269b0dac9c15b7e68adfa68aa4d
3
+ size 4214714413
model-00150-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc2e295d9b8d47e3ca2e7e3adedb53843b1fa5b22ac3f26e75da7069d8e2656b
3
+ size 3925868951
model-00155-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e6df72b8741383c068b1d7116f73261140597109a44d50028d93ff34ebb5947
3
+ size 3925868957
model-00159-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aaac2193f58291adaa806e4771efd7a11c48ddf2a26dfd9364f110b888e28389
3
+ size 3925868951
model-00161-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0523e0451fad72c5bf12ed4bbfc9d7801e83a568b15675e97223c30b9d18448
3
+ size 3925868957
model-00162-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83324d8fd2f35f7d44893b7b1d52d066b4ab5bc7fd422565ff39c8ef2c42c48c
3
+ size 3925868951
model-00164-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d57930fcee03a4c5080565747a8882bd854038b1d6048b0cb56f321c57119fd
3
+ size 3925868957
model-00167-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:842e5778b93a0886deedaf6266a9bbe9c49dc4b87457b8d13c86dd2cc3e4f43b
3
+ size 3925868957
model-00168-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39003f8860260c7f965727a7d6cd7c43a73b49e8fab416f4bf5ba262ccbccce8
3
+ size 3925868951
model-00191-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b66e929d9717ca920ea883bd7c2c4c9f2b0b098d492ee9863ddba33456d0210d
3
+ size 3925868957
model-00192-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:80f285a8b534f54bb4179ff107037eb1f5882071a455f7d5d76e4517e0110a6c
3
+ size 3925868951
model-00194-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9dcdfb9d45b3d3bc329c1924e5badd7112a012295dc74eb90d80612e55fdcc8
3
+ size 3925868957
model-00197-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21277aa7f841d793406f9b07a5aecda6ce8dfa6fb13108e5b8af9466da91958c
3
+ size 3925868957
model-00203-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:937f5af53d0ddb62c198ec9870a06bc89d4556e74161bc327fad1a2e7e95105b
3
+ size 3925868957
model-00206-of-00208.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c98c14d1545fad119bbfeed3ac6ee3d850c9d6c2c572629cf5a7efec3ea73614
3
+ size 3925868957
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
modeling_mimo_v2.py ADDED
@@ -0,0 +1,697 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ #
3
+ # Copyright 2026 Xiaomi Corporation.
4
+ # Copyright 2026 The HuggingFace Inc. team.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+
18
+ from copy import copy
19
+ from typing import Callable, Optional, Union
20
+
21
+ import torch
22
+ import torch.nn as nn
23
+ import torch.nn.functional as F
24
+
25
+ from transformers.activations import ACT2FN
26
+ from transformers.cache_utils import Cache, DynamicCache
27
+ from transformers.generation import GenerationMixin
28
+ from transformers.integrations import use_kernel_forward_from_hub
29
+ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
30
+ from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
31
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
32
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
33
+ from transformers.processing_utils import Unpack
34
+ from transformers.utils import TransformersKwargs, can_return_tuple, logging
35
+
36
+ from .configuration_mimo_v2 import MiMoV2Config
37
+
38
+
39
+ logger = logging.get_logger(__name__)
40
+
41
+
42
+ def rotate_half(x):
43
+ """Rotates half the hidden dims of the input."""
44
+ x1 = x[..., : x.shape[-1] // 2]
45
+ x2 = x[..., x.shape[-1] // 2 :]
46
+ return torch.cat((-x2, x1), dim=-1)
47
+
48
+
49
+ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
50
+ """Applies rotary position embedding to query and key tensors."""
51
+ cos = cos.unsqueeze(unsqueeze_dim)
52
+ sin = sin.unsqueeze(unsqueeze_dim)
53
+ q_embed = (q * cos) + (rotate_half(q) * sin)
54
+ k_embed = (k * cos) + (rotate_half(k) * sin)
55
+ return q_embed, k_embed
56
+
57
+
58
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
59
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
60
+ if n_rep == 1:
61
+ return hidden_states
62
+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
63
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
64
+
65
+
66
+ def eager_attention_forward(
67
+ module: nn.Module,
68
+ query: torch.Tensor,
69
+ key: torch.Tensor,
70
+ value: torch.Tensor,
71
+ attention_mask: Optional[torch.Tensor],
72
+ scaling: float,
73
+ dropout: float = 0.0,
74
+ sinks: Optional[torch.Tensor] = None,
75
+ **kwargs,
76
+ ):
77
+ key_states = repeat_kv(key, module.num_key_value_groups)
78
+ value_states = repeat_kv(value, module.num_key_value_groups)
79
+ attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
80
+ if attention_mask is not None:
81
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
82
+ attn_weights = attn_weights + causal_mask
83
+
84
+ if sinks is not None:
85
+ sinks = module.attention_sink_bias.reshape(1, -1, 1, 1).expand(query.shape[0], -1, query.shape[-2], -1)
86
+ attn_weights = torch.cat([attn_weights, sinks], dim=-1)
87
+
88
+ attn_weights = attn_weights - attn_weights.max(dim=-1, keepdim=True).values
89
+ probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
90
+
91
+ if sinks is not None:
92
+ probs = probs[..., :-1]
93
+
94
+ attn_weights = nn.functional.dropout(probs, p=dropout, training=module.training)
95
+ attn_output = torch.matmul(attn_weights, value_states)
96
+ attn_output = attn_output.transpose(1, 2).contiguous()
97
+ return attn_output, attn_weights
98
+
99
+
100
+ @use_kernel_forward_from_hub("RMSNorm")
101
+ class MiMoV2RMSNorm(nn.Module):
102
+ def __init__(self, hidden_size, eps=1e-6):
103
+ super().__init__()
104
+ self.weight = nn.Parameter(torch.ones(hidden_size))
105
+ self.variance_epsilon = eps
106
+
107
+ def forward(self, hidden_states):
108
+ input_dtype = hidden_states.dtype
109
+ hidden_states = hidden_states.to(torch.float32)
110
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
111
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
112
+ return self.weight * hidden_states.to(input_dtype)
113
+
114
+
115
+ class MiMoV2MLP(nn.Module):
116
+ def __init__(self, config, intermediate_size=None):
117
+ super().__init__()
118
+ self.config = config
119
+ self.hidden_size = config.hidden_size
120
+ self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
121
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
122
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
123
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
124
+ self.act_fn = ACT2FN[config.hidden_act]
125
+
126
+ def forward(self, hidden_states):
127
+ return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
128
+
129
+
130
+ class MiMoV2MoEGate(nn.Module):
131
+ def __init__(self, config):
132
+ super().__init__()
133
+ self.config = config
134
+ self.top_k = config.num_experts_per_tok
135
+ self.n_routed_experts = config.n_routed_experts
136
+ self.routed_scaling_factor = config.routed_scaling_factor if config.routed_scaling_factor is not None else 1.0
137
+ self.scoring_func = config.scoring_func
138
+ self.topk_method = config.topk_method
139
+ self.n_group = config.n_group
140
+ self.topk_group = config.topk_group
141
+ self.norm_topk_prob = config.norm_topk_prob
142
+ self.gating_dim = config.hidden_size
143
+ self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
144
+ if self.topk_method == "noaux_tc":
145
+ self.e_score_correction_bias = nn.Parameter(torch.empty((self.n_routed_experts)))
146
+
147
+ def forward(self, hidden_states):
148
+ bsz, seq_len, h = hidden_states.shape
149
+ hidden_states = hidden_states.view(-1, h)
150
+ logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32), None)
151
+ if self.scoring_func == "sigmoid":
152
+ scores = logits.sigmoid()
153
+ else:
154
+ raise NotImplementedError(f"Unsupported scoring function for MoE gating: {self.scoring_func}")
155
+
156
+ if self.topk_method == "noaux_tc":
157
+ if self.training:
158
+ raise ValueError("MiMoV2 noaux_tc routing is only implemented for inference.")
159
+ scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)
160
+ group_scores = scores_for_choice.view(bsz * seq_len, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
161
+ group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
162
+ group_mask = torch.zeros_like(group_scores)
163
+ group_mask.scatter_(1, group_idx, 1)
164
+ score_mask = (
165
+ group_mask.unsqueeze(-1)
166
+ .expand(bsz * seq_len, self.n_group, self.n_routed_experts // self.n_group)
167
+ .reshape(bsz * seq_len, -1)
168
+ )
169
+ tmp_scores = scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))
170
+ _, topk_idx = torch.topk(tmp_scores, k=self.top_k, dim=-1, sorted=False)
171
+ topk_weight = scores.gather(1, topk_idx)
172
+ else:
173
+ raise NotImplementedError(f"Unsupported TopK function for MoE gating: {self.topk_method}")
174
+
175
+ if self.top_k > 1 and self.norm_topk_prob:
176
+ denominator = topk_weight.sum(dim=-1, keepdim=True) + 1e-20
177
+ topk_weight = topk_weight / denominator
178
+ topk_weight = topk_weight * self.routed_scaling_factor
179
+ return topk_idx, topk_weight
180
+
181
+
182
+ class MiMoV2MoE(nn.Module):
183
+ def __init__(self, config):
184
+ super().__init__()
185
+ self.config = config
186
+ self.experts = nn.ModuleList(
187
+ [MiMoV2MLP(config, intermediate_size=config.moe_intermediate_size) for _ in range(config.n_routed_experts)]
188
+ )
189
+ self.gate = MiMoV2MoEGate(config)
190
+
191
+ def moe(self, hidden_states: torch.Tensor, topk_indices: torch.Tensor, topk_weights: torch.Tensor):
192
+ final_hidden_states = torch.zeros_like(hidden_states, dtype=topk_weights.dtype)
193
+ expert_mask = torch.nn.functional.one_hot(topk_indices, num_classes=len(self.experts))
194
+ expert_mask = expert_mask.permute(2, 0, 1)
195
+
196
+ for expert_idx, expert in enumerate(self.experts):
197
+ mask = expert_mask[expert_idx]
198
+ token_indices, weight_indices = torch.where(mask)
199
+ if token_indices.numel() > 0:
200
+ expert_weights = topk_weights[token_indices, weight_indices]
201
+ expert_input = hidden_states[token_indices]
202
+ expert_output = expert(expert_input)
203
+ final_hidden_states.index_add_(0, token_indices, expert_output * expert_weights.unsqueeze(-1))
204
+
205
+ return final_hidden_states.type(hidden_states.dtype)
206
+
207
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
208
+ orig_shape = hidden_states.shape
209
+ topk_indices, topk_weights = self.gate(hidden_states)
210
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
211
+ hidden_states = self.moe(hidden_states, topk_indices, topk_weights).view(*orig_shape)
212
+ return hidden_states
213
+
214
+
215
+ class MiMoV2Attention(nn.Module):
216
+ """MiMoV2 attention.
217
+
218
+ `projection_layout` only controls how checkpoint weights are named and
219
+ stored: Flash uses separate q/k/v projections, while Pro uses fused qkv.
220
+ The attention computation after projection is shared.
221
+ """
222
+
223
+ def __init__(self, config, is_swa: bool, layer_idx: int, projection_layout: str = "split"):
224
+ super().__init__()
225
+ if projection_layout not in {"split", "fused_qkv"}:
226
+ raise ValueError(f"Unsupported MiMoV2 attention projection layout: {projection_layout}")
227
+
228
+ self.config = config
229
+ self.layer_idx = layer_idx
230
+ self.is_swa = is_swa
231
+ self.is_causal = True
232
+ self.projection_layout = projection_layout
233
+
234
+ default_head_dim = config.hidden_size // config.num_attention_heads
235
+ default_v_head_dim = getattr(config, "v_head_dim", default_head_dim)
236
+
237
+ if is_swa:
238
+ self.head_dim = getattr(config, "swa_head_dim", getattr(config, "head_dim", default_head_dim))
239
+ self.v_head_dim = getattr(config, "swa_v_head_dim", default_v_head_dim)
240
+ self.num_attention_heads = getattr(config, "swa_num_attention_heads", config.num_attention_heads)
241
+ self.num_key_value_heads = getattr(config, "swa_num_key_value_heads", config.num_key_value_heads)
242
+ else:
243
+ self.head_dim = getattr(config, "head_dim", default_head_dim)
244
+ self.v_head_dim = getattr(config, "v_head_dim", self.head_dim)
245
+ self.num_attention_heads = config.num_attention_heads
246
+ self.num_key_value_heads = config.num_key_value_heads
247
+
248
+ self.rope_dim = int(self.head_dim * getattr(config, "partial_rotary_factor", 1.0))
249
+ if self.rope_dim % 2 != 0:
250
+ raise ValueError(
251
+ f"MiMoV2 rotary dimension must be even, got {self.rope_dim} from "
252
+ f"head_dim={self.head_dim} and partial_rotary_factor={getattr(config, 'partial_rotary_factor', 1.0)}"
253
+ )
254
+ self.num_key_value_groups = self.num_attention_heads // self.num_key_value_heads
255
+ self.attention_dropout = getattr(config, "attention_dropout", 0.0)
256
+ self.scaling = self.head_dim**-0.5
257
+ self.sliding_window = getattr(config, "sliding_window", None) if is_swa else None
258
+ self.q_size = self.num_attention_heads * self.head_dim
259
+ self.k_size = self.num_key_value_heads * self.head_dim
260
+ self.v_size = self.num_key_value_heads * self.v_head_dim
261
+ self.o_hidden_size = self.num_attention_heads * self.v_head_dim
262
+ self.v_scale = getattr(config, "attention_value_scale", None)
263
+ self.attention_sink_bias = (
264
+ nn.Parameter(torch.empty(self.num_attention_heads), requires_grad=False)
265
+ if (
266
+ (getattr(config, "add_full_attention_sink_bias", False) and not is_swa)
267
+ or (getattr(config, "add_swa_attention_sink_bias", False) and is_swa)
268
+ )
269
+ else None
270
+ )
271
+
272
+ attention_bias = getattr(config, "attention_bias", False)
273
+ if self.projection_layout == "fused_qkv":
274
+ self.qkv_proj = nn.Linear(
275
+ config.hidden_size,
276
+ self.q_size + self.k_size + self.v_size,
277
+ bias=attention_bias,
278
+ )
279
+ else:
280
+ self.q_proj = nn.Linear(config.hidden_size, self.q_size, bias=attention_bias)
281
+ self.k_proj = nn.Linear(config.hidden_size, self.k_size, bias=attention_bias)
282
+ self.v_proj = nn.Linear(config.hidden_size, self.v_size, bias=attention_bias)
283
+ self.o_proj = nn.Linear(self.o_hidden_size, config.hidden_size, bias=False)
284
+
285
+ def _forward_attention(
286
+ self,
287
+ query_states: torch.Tensor,
288
+ key_states: torch.Tensor,
289
+ value_states: torch.Tensor,
290
+ input_shape: torch.Size,
291
+ position_embeddings: tuple[torch.Tensor, torch.Tensor],
292
+ attention_mask: Optional[torch.Tensor],
293
+ past_key_values: Optional[Cache] = None,
294
+ cache_position: Optional[torch.LongTensor] = None,
295
+ position_ids: Optional[torch.LongTensor] = None,
296
+ ) -> tuple[torch.Tensor, torch.Tensor]:
297
+ if self.v_scale is not None:
298
+ value_states = value_states * self.v_scale
299
+
300
+ cos, sin = position_embeddings
301
+ query_rope, query_nope = query_states.split([self.rope_dim, self.head_dim - self.rope_dim], dim=-1)
302
+ key_rope, key_nope = key_states.split([self.rope_dim, self.head_dim - self.rope_dim], dim=-1)
303
+ query_rope, key_rope = apply_rotary_pos_emb(query_rope, key_rope, cos, sin)
304
+ query_states = torch.cat([query_rope, query_nope], dim=-1)
305
+ key_states = torch.cat([key_rope, key_nope], dim=-1)
306
+
307
+ if past_key_values is not None:
308
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
309
+ key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
310
+
311
+ attn_implementation = self.config._attn_implementation
312
+ if attn_implementation is not None and attn_implementation.startswith("paged|"):
313
+ raise ValueError(
314
+ "MiMoV2 remote code does not support paged attention cache. "
315
+ "Please use eager, sdpa, flex_attention, or flash_attention_2."
316
+ )
317
+
318
+ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
319
+ attn_implementation, eager_attention_forward
320
+ )
321
+ if self.attention_sink_bias is not None and attn_implementation == "sdpa":
322
+ logger.warning_once(
323
+ "MiMoV2 attention sink bias is not supported by SDPA; falling back to eager attention for correctness."
324
+ )
325
+ attention_interface = eager_attention_forward
326
+
327
+ attention_kwargs = {
328
+ "dropout": 0.0 if not self.training else self.attention_dropout,
329
+ "scaling": self.scaling,
330
+ "position_ids": position_ids,
331
+ "is_causal": self.is_causal,
332
+ }
333
+ if attention_interface is eager_attention_forward:
334
+ attention_kwargs["sinks"] = self.attention_sink_bias
335
+ else:
336
+ if self.attention_sink_bias is not None:
337
+ attention_kwargs["s_aux"] = self.attention_sink_bias
338
+ if self.sliding_window is not None:
339
+ attention_kwargs["sliding_window"] = self.sliding_window
340
+
341
+ attn_output, attn_weights = attention_interface(
342
+ self,
343
+ query_states,
344
+ key_states,
345
+ value_states,
346
+ attention_mask,
347
+ **attention_kwargs,
348
+ )
349
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
350
+ attn_output = self.o_proj(attn_output)
351
+ return attn_output, attn_weights
352
+
353
+ def forward(
354
+ self,
355
+ hidden_states: torch.Tensor,
356
+ position_embeddings: tuple[torch.Tensor, torch.Tensor],
357
+ attention_mask: Optional[torch.Tensor],
358
+ past_key_values: Optional[Cache] = None,
359
+ cache_position: Optional[torch.LongTensor] = None,
360
+ position_ids: Optional[torch.LongTensor] = None,
361
+ **kwargs: Unpack[TransformersKwargs],
362
+ ) -> tuple[torch.Tensor, torch.Tensor]:
363
+ input_shape = hidden_states.shape[:-1]
364
+
365
+ if self.projection_layout == "fused_qkv":
366
+ qkv_states = self.qkv_proj(hidden_states)
367
+ query_states, key_states, value_states = qkv_states.split([self.q_size, self.k_size, self.v_size], dim=-1)
368
+ else:
369
+ query_states = self.q_proj(hidden_states)
370
+ key_states = self.k_proj(hidden_states)
371
+ value_states = self.v_proj(hidden_states)
372
+
373
+ query_states = query_states.view(*input_shape, self.num_attention_heads, self.head_dim).transpose(1, 2)
374
+ key_states = key_states.view(*input_shape, self.num_key_value_heads, self.head_dim).transpose(1, 2)
375
+ value_states = value_states.view(*input_shape, self.num_key_value_heads, self.v_head_dim).transpose(1, 2)
376
+ return self._forward_attention(
377
+ query_states,
378
+ key_states,
379
+ value_states,
380
+ input_shape,
381
+ position_embeddings,
382
+ attention_mask,
383
+ past_key_values=past_key_values,
384
+ cache_position=cache_position,
385
+ position_ids=position_ids,
386
+ )
387
+
388
+
389
+ class MiMoV2DecoderLayer(nn.Module):
390
+ attention_projection_layout = "split"
391
+
392
+ def __init__(self, config, layer_idx: int, attention_projection_layout: Optional[str] = None):
393
+ super().__init__()
394
+ attention_projection_layout = attention_projection_layout or self.attention_projection_layout
395
+ is_swa_layer = config.hybrid_layer_pattern[layer_idx] == 1
396
+ self.attention_type = "sliding_window_attention" if is_swa_layer else "full_attention"
397
+ self.self_attn = MiMoV2Attention(
398
+ config, is_swa_layer, layer_idx, projection_layout=attention_projection_layout
399
+ )
400
+ self.mlp = (
401
+ MiMoV2MoE(config)
402
+ if getattr(config, "n_routed_experts", None) is not None and config.moe_layer_freq[layer_idx]
403
+ else MiMoV2MLP(config)
404
+ )
405
+ self.input_layernorm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
406
+ self.post_attention_layernorm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
407
+
408
+ def forward(
409
+ self,
410
+ hidden_states: torch.Tensor,
411
+ attention_mask: Optional[torch.Tensor] = None,
412
+ position_ids: Optional[torch.LongTensor] = None,
413
+ past_key_values: Optional[Cache] = None,
414
+ use_cache: Optional[bool] = False,
415
+ cache_position: Optional[torch.LongTensor] = None,
416
+ position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
417
+ **kwargs: Unpack[TransformersKwargs],
418
+ ) -> torch.Tensor:
419
+ residual = hidden_states
420
+ hidden_states = self.input_layernorm(hidden_states)
421
+ hidden_states, _ = self.self_attn(
422
+ hidden_states=hidden_states,
423
+ attention_mask=attention_mask,
424
+ position_ids=position_ids,
425
+ past_key_values=past_key_values,
426
+ use_cache=use_cache,
427
+ cache_position=cache_position,
428
+ position_embeddings=position_embeddings,
429
+ **kwargs,
430
+ )
431
+ hidden_states = residual + hidden_states
432
+
433
+ residual = hidden_states
434
+ hidden_states = self.post_attention_layernorm(hidden_states)
435
+ hidden_states = self.mlp(hidden_states)
436
+ hidden_states = residual + hidden_states
437
+ return hidden_states
438
+
439
+
440
+ class MiMoV2RotaryEmbedding(nn.Module):
441
+ inv_freq: torch.Tensor
442
+
443
+ def __init__(self, config, is_swa: bool, device=None):
444
+ super().__init__()
445
+ if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
446
+ self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type", "default"))
447
+ else:
448
+ self.rope_type = "default"
449
+ self.max_seq_len_cached = config.max_position_embeddings
450
+ self.original_max_seq_len = config.max_position_embeddings
451
+
452
+ self.config = copy(config)
453
+ self.config.rope_parameters = copy(getattr(config, "rope_parameters", None) or {})
454
+ if is_swa:
455
+ self.config.rope_theta = getattr(config, "swa_rope_theta", config.rope_theta)
456
+ self.config.head_dim = getattr(config, "swa_head_dim", getattr(config, "head_dim", None))
457
+ if self.config.rope_parameters:
458
+ self.config.rope_parameters["rope_theta"] = self.config.rope_theta
459
+ self.rope_init_fn = (
460
+ self.compute_default_rope_parameters
461
+ if self.rope_type == "default"
462
+ else ROPE_INIT_FUNCTIONS[self.rope_type]
463
+ )
464
+
465
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
466
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
467
+ self.original_inv_freq = self.inv_freq
468
+
469
+ @staticmethod
470
+ def compute_default_rope_parameters(config, device=None, seq_len=None, layer_type=None):
471
+ config.standardize_rope_params()
472
+ rope_parameters = config.rope_parameters[layer_type] if layer_type is not None else config.rope_parameters
473
+ base = rope_parameters["rope_theta"]
474
+ partial_rotary_factor = rope_parameters.get("partial_rotary_factor", 1.0)
475
+ head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
476
+ dim = int(head_dim * partial_rotary_factor)
477
+ if dim % 2 != 0:
478
+ raise ValueError(
479
+ f"MiMoV2 rotary dimension must be even, got {dim} from "
480
+ f"head_dim={head_dim} and partial_rotary_factor={partial_rotary_factor}"
481
+ )
482
+ inv_freq = 1.0 / (
483
+ base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)
484
+ )
485
+ return inv_freq, 1.0
486
+
487
+ @torch.no_grad()
488
+ @dynamic_rope_update
489
+ def forward(self, x, position_ids):
490
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
491
+ position_ids_expanded = position_ids[:, None, :].float()
492
+
493
+ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
494
+ with torch.autocast(device_type=device_type, enabled=False):
495
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
496
+ emb = torch.cat((freqs, freqs), dim=-1)
497
+ cos = emb.cos() * self.attention_scaling
498
+ sin = emb.sin() * self.attention_scaling
499
+
500
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
501
+
502
+
503
+ class MiMoV2Model(PreTrainedModel):
504
+ config_class = MiMoV2Config
505
+ attention_projection_layout = "split"
506
+
507
+ def __init__(self, config):
508
+ super().__init__(config)
509
+ self.attention_projection_layout = getattr(
510
+ config, "attention_projection_layout", self.attention_projection_layout
511
+ )
512
+ self.vocab_size = config.vocab_size
513
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
514
+ self.layers = nn.ModuleList(
515
+ [
516
+ MiMoV2DecoderLayer(
517
+ config,
518
+ layer_idx,
519
+ attention_projection_layout=self.attention_projection_layout,
520
+ )
521
+ for layer_idx in range(config.num_hidden_layers)
522
+ ]
523
+ )
524
+ self.norm = MiMoV2RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
525
+ self.rotary_emb = MiMoV2RotaryEmbedding(config=config, is_swa=False)
526
+ self.swa_rotary_emb = MiMoV2RotaryEmbedding(config=config, is_swa=True)
527
+ self.has_sliding_layers = any(pattern == 1 for pattern in config.hybrid_layer_pattern)
528
+ self.config.layer_types = [
529
+ "sliding_attention" if config.hybrid_layer_pattern[i] == 1 else "full_attention"
530
+ for i in range(config.num_hidden_layers)
531
+ ]
532
+ self.post_init()
533
+
534
+ def get_input_embeddings(self):
535
+ return self.embed_tokens
536
+
537
+ def set_input_embeddings(self, value):
538
+ self.embed_tokens = value
539
+
540
+ def forward(
541
+ self,
542
+ input_ids: Optional[torch.LongTensor] = None,
543
+ attention_mask: Optional[torch.Tensor] = None,
544
+ position_ids: Optional[torch.LongTensor] = None,
545
+ past_key_values: Optional[Cache] = None,
546
+ inputs_embeds: Optional[torch.FloatTensor] = None,
547
+ use_cache: Optional[bool] = None,
548
+ cache_position: Optional[torch.LongTensor] = None,
549
+ **kwargs: Unpack[TransformersKwargs],
550
+ ) -> BaseModelOutputWithPast:
551
+ if (input_ids is None) ^ (inputs_embeds is not None):
552
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
553
+
554
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
555
+
556
+ if inputs_embeds is None:
557
+ inputs_embeds = self.embed_tokens(input_ids)
558
+
559
+ if use_cache and past_key_values is None:
560
+ past_key_values = DynamicCache(config=self.config)
561
+
562
+ if cache_position is None:
563
+ past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
564
+ cache_position = torch.arange(
565
+ past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
566
+ )
567
+
568
+ if position_ids is None:
569
+ position_ids = cache_position.unsqueeze(0)
570
+
571
+ if not isinstance(causal_mask_mapping := attention_mask, dict):
572
+ mask_kwargs = {
573
+ "config": self.config,
574
+ "input_embeds": inputs_embeds,
575
+ "attention_mask": attention_mask,
576
+ "cache_position": cache_position,
577
+ "past_key_values": past_key_values,
578
+ "position_ids": position_ids,
579
+ }
580
+ causal_mask_mapping = {
581
+ "full_attention": create_causal_mask(**mask_kwargs),
582
+ }
583
+ if self.has_sliding_layers:
584
+ if getattr(self.config, "sliding_window", None) is None:
585
+ raise ValueError("MiMoV2 config `sliding_window` must be set when hybrid_layer_pattern uses SWA.")
586
+ causal_mask_mapping["sliding_window_attention"] = create_sliding_window_causal_mask(**mask_kwargs)
587
+
588
+ hidden_states = inputs_embeds
589
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
590
+ swa_position_embeddings = self.swa_rotary_emb(hidden_states, position_ids)
591
+
592
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
593
+ hidden_states = decoder_layer(
594
+ hidden_states,
595
+ attention_mask=causal_mask_mapping[decoder_layer.attention_type],
596
+ position_embeddings=position_embeddings
597
+ if decoder_layer.attention_type == "full_attention"
598
+ else swa_position_embeddings,
599
+ position_ids=position_ids,
600
+ past_key_values=past_key_values,
601
+ use_cache=use_cache,
602
+ cache_position=cache_position,
603
+ **kwargs,
604
+ )
605
+
606
+ hidden_states = self.norm(hidden_states)
607
+ return BaseModelOutputWithPast(
608
+ last_hidden_state=hidden_states,
609
+ past_key_values=past_key_values if use_cache else None,
610
+ )
611
+
612
+
613
+ class MiMoV2ForCausalLM(PreTrainedModel, GenerationMixin):
614
+ config_class = MiMoV2Config
615
+ model_class = MiMoV2Model
616
+ _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
617
+ _tp_plan = {"lm_head": "colwise_rep"}
618
+ _pp_plan = {"lm_head": (["hidden_states"], ["logits"])}
619
+ _keys_to_ignore_on_load_unexpected = [
620
+ r"model\.(swa_)?rotary_emb\.inv_freq",
621
+ r"model\.layers\.\d+\.self_attn\.rotary_emb\.inv_freq",
622
+ r"model\.layers\.\d+\.self_attn\.rotary_emb\.(cos_cached|sin_cached)",
623
+ r"model\.mtp\..*",
624
+ ]
625
+
626
+ def __init__(self, config):
627
+ super().__init__(config)
628
+ self.model = self.model_class(config)
629
+ self.vocab_size = config.vocab_size
630
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
631
+ self.post_init()
632
+
633
+ def get_input_embeddings(self):
634
+ return self.model.embed_tokens
635
+
636
+ def set_input_embeddings(self, value):
637
+ self.model.embed_tokens = value
638
+
639
+ def get_output_embeddings(self):
640
+ return self.lm_head
641
+
642
+ def set_output_embeddings(self, new_embeddings):
643
+ self.lm_head = new_embeddings
644
+
645
+ @can_return_tuple
646
+ def forward(
647
+ self,
648
+ input_ids: Optional[torch.LongTensor] = None,
649
+ attention_mask: Optional[torch.Tensor] = None,
650
+ position_ids: Optional[torch.LongTensor] = None,
651
+ past_key_values: Optional[Cache] = None,
652
+ inputs_embeds: Optional[torch.FloatTensor] = None,
653
+ labels: Optional[torch.LongTensor] = None,
654
+ use_cache: Optional[bool] = None,
655
+ cache_position: Optional[torch.LongTensor] = None,
656
+ logits_to_keep: Union[int, torch.Tensor] = 0,
657
+ **kwargs: Unpack[TransformersKwargs],
658
+ ) -> CausalLMOutputWithPast:
659
+ outputs: BaseModelOutputWithPast = self.model(
660
+ input_ids=input_ids,
661
+ attention_mask=attention_mask,
662
+ position_ids=position_ids,
663
+ past_key_values=past_key_values,
664
+ inputs_embeds=inputs_embeds,
665
+ use_cache=use_cache,
666
+ cache_position=cache_position,
667
+ **kwargs,
668
+ )
669
+
670
+ hidden_states = outputs.last_hidden_state
671
+ slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
672
+ logits = self.lm_head(hidden_states[:, slice_indices, :])
673
+
674
+ loss = None
675
+ if labels is not None:
676
+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
677
+
678
+ return CausalLMOutputWithPast(
679
+ loss=loss,
680
+ logits=logits,
681
+ past_key_values=outputs.past_key_values,
682
+ hidden_states=outputs.hidden_states,
683
+ attentions=outputs.attentions,
684
+ )
685
+
686
+
687
+ __all__ = [
688
+ "MiMoV2Attention",
689
+ "MiMoV2DecoderLayer",
690
+ "MiMoV2ForCausalLM",
691
+ "MiMoV2MLP",
692
+ "MiMoV2MoE",
693
+ "MiMoV2MoEGate",
694
+ "MiMoV2Model",
695
+ "MiMoV2RMSNorm",
696
+ "MiMoV2RotaryEmbedding",
697
+ ]
tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "is_local": true,
9
+ "model_max_length": 131272,
10
+ "pad_token": "<|endoftext|>",
11
+ "split_special_tokens": false,
12
+ "tokenizer_class": "Qwen2Tokenizer",
13
+ "tool_parser_type": "qwen3_coder",
14
+ "unk_token": null
15
+ }