[Fix] Plan NextN / MTP draft layers as one-layer models and fix the Bailing V2 NextN draft (#41194)
This commit is contained in:
@@ -1283,9 +1283,12 @@ class ModelConfig:
|
||||
self.v_head_dim = tc.v_head_dim
|
||||
self.qk_nope_head_dim = tc.qk_nope_head_dim
|
||||
self._init_mla_scaling(getattr(tc, "rope_scaling", None))
|
||||
elif (
|
||||
"BailingMoeV2_5ForCausalLM" in self.hf_config.architectures
|
||||
or "BailingMoeForCausalLMNextN" in self.hf_config.architectures
|
||||
elif "BailingMoeV2_5ForCausalLM" in self.hf_config.architectures or (
|
||||
# Every Bailing draft is renamed to BailingMoeForCausalLMNextN; only
|
||||
# the MLA ones (V2.5 and V3) carry kv_lora_rank, V2 keeps its GQA
|
||||
# shapes.
|
||||
"BailingMoeForCausalLMNextN" in self.hf_config.architectures
|
||||
and getattr(self.hf_text_config, "kv_lora_rank", None) is not None
|
||||
):
|
||||
self.head_dim = self.hf_text_config.head_dim
|
||||
self.attention_arch = AttentionArch.MLA
|
||||
|
||||
@@ -660,7 +660,7 @@ class BailingMoEBlock(nn.Module):
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id, is_nextn=False
|
||||
config, layer_id=layer_id, is_nextn=is_nextn
|
||||
)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id - 1, is_nextn=False
|
||||
@@ -671,7 +671,8 @@ class BailingMoEBlock(nn.Module):
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
# A NextN draft is a one-layer model.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_layer_sparse,
|
||||
is_previous_layer_sparse=is_previous_layer_sparse,
|
||||
is_next_layer_sparse=is_next_layer_sparse,
|
||||
|
||||
@@ -781,7 +781,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
|
||||
self.expert_num = config.num_experts
|
||||
self.hidden_size = config.hidden_size
|
||||
is_moe_layer = self._is_layer_sparse(config, self.layer_id)
|
||||
is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn)
|
||||
is_previous_moe_layer = self._is_layer_sparse(config, self.layer_id - 1)
|
||||
is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1)
|
||||
if self.expert_num == 1:
|
||||
@@ -815,7 +815,8 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=config.num_hidden_layers,
|
||||
# A NextN draft is a one-layer model.
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=is_moe_layer,
|
||||
is_previous_layer_sparse=is_previous_moe_layer,
|
||||
is_next_layer_sparse=is_next_layer_moe_layer,
|
||||
|
||||
@@ -562,11 +562,10 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
# An MTP draft is a one-layer model; layer_id still indexes its config.
|
||||
self.layer_scatter_modes = LayerScatterModes.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=(
|
||||
config.num_hidden_layers if layer_id < config.num_hidden_layers else 1
|
||||
), # 1 is for mtp
|
||||
layer_id=0 if is_nextn else layer_id,
|
||||
num_layers=1 if is_nextn else config.num_hidden_layers,
|
||||
is_layer_sparse=self.is_moe_layer,
|
||||
is_previous_layer_sparse=self.is_previous_layer_sparse,
|
||||
is_next_layer_sparse=self.is_next_layer_sparse,
|
||||
|
||||
Reference in New Issue
Block a user