Upload modeling_bailing_moe_v2.py with huggingface_hub
Browse files
modeling_bailing_moe_v2.py
CHANGED
@@ -1159,8 +1159,8 @@ class BailingMoeV2Model(BailingMoeV2PreTrainedModel):
|
|
1159 |
|
1160 |
self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
1161 |
self.layers = []
|
1162 |
-
for layer_idx in range(config.num_hidden_layers
|
1163 |
-
layer_cls = BailingMoeV2DecoderLayer if layer_idx < config.num_hidden_layers else BailingMoeV2MTPLayer
|
1164 |
self.layers.append(layer_cls(config, layer_idx))
|
1165 |
|
1166 |
self.layers = nn.ModuleList(self.layers)
|
|
|
1159 |
|
1160 |
self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
1161 |
self.layers = []
|
1162 |
+
for layer_idx in range(config.num_hidden_layers):
|
1163 |
+
layer_cls = BailingMoeV2DecoderLayer if layer_idx < config.num_hidden_layers - config.num_nextn_predict_layers else BailingMoeV2MTPLayer
|
1164 |
self.layers.append(layer_cls(config, layer_idx))
|
1165 |
|
1166 |
self.layers = nn.ModuleList(self.layers)
|