Update modeling_qwen3sa.py
Browse files- modeling_qwen3sa.py +1 -1
modeling_qwen3sa.py
CHANGED
|
@@ -706,7 +706,7 @@ class Qwen3SummaryAttention(Qwen3Attention):
|
|
| 706 |
else:
|
| 707 |
self._sliding_chunk_num = int(val)
|
| 708 |
|
| 709 |
-
if config.summary_independent_parameters:
|
| 710 |
self.q_proj_summary = nn.Linear(
|
| 711 |
config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
|
| 712 |
)
|
|
|
|
| 706 |
else:
|
| 707 |
self._sliding_chunk_num = int(val)
|
| 708 |
|
| 709 |
+
if config.summary_independent_parameters and config.mix_coeff > 0:
|
| 710 |
self.q_proj_summary = nn.Linear(
|
| 711 |
config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
|
| 712 |
)
|