Update modeling_bailing_moe_v2_5.py

#6
Files changed (1) hide show
  1. modeling_bailing_moe_v2_5.py +1 -1
modeling_bailing_moe_v2_5.py CHANGED
@@ -752,7 +752,7 @@ class BailingMoeV2_5LinearAttention(nn.Module):
752
  self.num_heads * self.head_dim, group_norm_size=config.group_norm_size, eps=self.rms_norm_eps
753
  )
754
  slope = -BailingMoeV2_5LinearAttention.build_slope_tensor(self.num_heads) * (
755
- 1 - (self.layer_idx - 1) / (self.config.num_hidden_layers - 1) + 1e-5
756
  )
757
  self.register_buffer('slope', slope, persistent=False)
758
 
 
752
  self.num_heads * self.head_dim, group_norm_size=config.group_norm_size, eps=self.rms_norm_eps
753
  )
754
  slope = -BailingMoeV2_5LinearAttention.build_slope_tensor(self.num_heads) * (
755
+ 1 - self.layer_idx / (self.config.num_hidden_layers - 1) + 1e-5
756
  )
757
  self.register_buffer('slope', slope, persistent=False)
758