zhanghanxiao commited on
Commit
3d203b4
·
verified ·
1 Parent(s): 1d1ad8c

Upload modeling_bailing_moe_v2.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. modeling_bailing_moe_v2.py +2 -2
modeling_bailing_moe_v2.py CHANGED
@@ -1159,8 +1159,8 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
1159
 
1160
  self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
1161
  self.layers = []
1162
- for layer_idx in range(config.num_hidden_layers + config.num_nextn_predict_layers):
1163
- layer_cls = BailingMoeV2DecoderLayer if layer_idx < config.num_hidden_layers else BailingMoeV2MTPLayer
1164
  self.layers.append(layer_cls(config, layer_idx))
1165
 
1166
  self.layers = nn.ModuleList(self.layers)
 
1159
 
1160
  self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
1161
  self.layers = []
1162
+ for layer_idx in range(config.num_hidden_layers):
1163
+ layer_cls = BailingMoeV2DecoderLayer if layer_idx < config.num_hidden_layers - config.num_nextn_predict_layers else BailingMoeV2MTPLayer
1164
  self.layers.append(layer_cls(config, layer_idx))
1165
 
1166
  self.layers = nn.ModuleList(self.layers)