Update modeling_bailing_moe_v2_5.py
#6
by kernelpool - opened
modeling_bailing_moe_v2_5.py
CHANGED
|
@@ -752,7 +752,7 @@ class BailingMoeV2_5LinearAttention(nn.Module):
|
|
| 752 |
self.num_heads * self.head_dim, group_norm_size=config.group_norm_size, eps=self.rms_norm_eps
|
| 753 |
)
|
| 754 |
slope = -BailingMoeV2_5LinearAttention.build_slope_tensor(self.num_heads) * (
|
| 755 |
-
1 -
|
| 756 |
)
|
| 757 |
self.register_buffer('slope', slope, persistent=False)
|
| 758 |
|
|
|
|
| 752 |
self.num_heads * self.head_dim, group_norm_size=config.group_norm_size, eps=self.rms_norm_eps
|
| 753 |
)
|
| 754 |
slope = -BailingMoeV2_5LinearAttention.build_slope_tensor(self.num_heads) * (
|
| 755 |
+
1 - self.layer_idx / (self.config.num_hidden_layers - 1) + 1e-5
|
| 756 |
)
|
| 757 |
self.register_buffer('slope', slope, persistent=False)
|
| 758 |
|