[Refactor] Drop model placement attributes and parameters nothing reads (#41812)
This commit is contained in:
@@ -158,7 +158,6 @@ class AfmoeMoE(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.rank = get_parallel().tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
|
||||
self.n_routed_experts = getattr(config, "num_experts", None)
|
||||
|
||||
@@ -329,9 +329,6 @@ class BailingMoESparseMoeBlock(nn.Module):
|
||||
)
|
||||
# dispatcher
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
# TODO: we will support tp < ep in the future
|
||||
self.ep_size = get_parallel().tp_size
|
||||
|
||||
self.deepep_dispatcher = DeepEPDispatcher(
|
||||
group=get_parallel().tp_group.device_group,
|
||||
router_topk=self.top_k,
|
||||
@@ -509,7 +506,6 @@ class BailingMoEAttention(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.total_kv_heads = config.num_key_value_heads
|
||||
self.dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
@@ -652,7 +648,6 @@ class BailingMoEBlock(nn.Module):
|
||||
hidden_size = config.hidden_size
|
||||
|
||||
self.input_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
self.dp_size = get_parallel().attn_dp_size
|
||||
self.attention = BailingMoEAttention(
|
||||
config,
|
||||
layer_id,
|
||||
@@ -662,8 +657,6 @@ class BailingMoEBlock(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
self.layer_id = layer_id
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(
|
||||
config, layer_id=layer_id, is_nextn=is_nextn
|
||||
|
||||
@@ -246,7 +246,6 @@ class BailingMoE(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.norm_expert_prob = getattr(config, "norm_topk_prob", False)
|
||||
|
||||
@@ -219,7 +219,6 @@ class BailingMoeForCausalLMNextN(nn.Module):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = 0
|
||||
is_bailing_moe_v3 = _is_bailing_moe_v3_config(config)
|
||||
|
||||
@@ -317,7 +317,6 @@ class BailingMLP(nn.Module):
|
||||
self.config = config
|
||||
self.swiglu_limit = swiglu_limit
|
||||
self.tp_size = tp_size if tp_size is not None else get_parallel().tp_size
|
||||
self.tp_rank = tp_rank if tp_rank is not None else get_parallel().tp_rank
|
||||
|
||||
self.intermediate_size = intermediate_size
|
||||
self.padded_intermediate_size = padded_intermediate_size or intermediate_size
|
||||
@@ -465,10 +464,8 @@ class BailingMoE(nn.Module):
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.moe_ep_size = get_parallel().moe_ep_size
|
||||
self.moe_tp_size = get_parallel().moe_tp_size
|
||||
self.moe_tp_rank = get_parallel().moe_tp_rank
|
||||
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.norm_expert_prob = getattr(config, "norm_topk_prob", False)
|
||||
@@ -1425,7 +1422,6 @@ class BailingMoeV3ForCausalLM(nn.Module):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
|
||||
# Determine num_fused_shared_experts
|
||||
self.determine_num_fused_shared_experts()
|
||||
|
||||
@@ -70,7 +70,6 @@ class DbrxRouter(nn.Module):
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.num_total_experts = config.ffn_config.moe_num_experts
|
||||
self.d_model = config.d_model
|
||||
self.layer = ReplicatedLinear(
|
||||
@@ -244,7 +243,6 @@ class DbrxAttention(nn.Module):
|
||||
)
|
||||
|
||||
tp_world_size = get_parallel().tp_size
|
||||
self.tp_size = tp_world_size
|
||||
assert self.total_num_heads % tp_world_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_world_size
|
||||
if self.total_num_kv_heads >= tp_world_size:
|
||||
|
||||
@@ -115,7 +115,6 @@ class DeepseekMoE(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.rank = get_parallel().tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.n_routed_experts = config.n_routed_experts
|
||||
self.top_k = config.num_experts_per_tok
|
||||
|
||||
@@ -277,7 +277,6 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
# if not set, model load will be broken in DeepseekV3ForCausalLM load_weights()
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
@@ -3241,7 +3241,6 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.use_dsa = is_deepseek_dsa(config)
|
||||
|
||||
@@ -1834,7 +1834,6 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.model = Dots3Model(
|
||||
@@ -2716,7 +2715,6 @@ class DotsNoteOmniForConditionalGeneration(nn.Module):
|
||||
)
|
||||
language_model = self.thinker.language_model
|
||||
self.pp_group = language_model.pp_group
|
||||
self.tp_size = language_model.tp_size
|
||||
self.quant_config = language_model.quant_config
|
||||
self.num_fused_shared_experts = language_model.num_fused_shared_experts
|
||||
self.forward = self.thinker.forward
|
||||
|
||||
@@ -146,7 +146,6 @@ class Dots3NoteForCausalLMNextN(Dots3LanguageModelForCausalLM):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.fuse_qkv_a_g_proj = True
|
||||
|
||||
@@ -244,9 +244,6 @@ class Exaone4DecoderLayer(nn.Module):
|
||||
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.self_attn = Exaone4Attention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
|
||||
@@ -134,7 +134,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.moe_ep_size = get_parallel().moe_ep_size
|
||||
self.layer_id = layer_id
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.alt_stream = alt_stream
|
||||
@@ -200,7 +199,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
)
|
||||
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.num_experts = (
|
||||
config.num_experts + get_exec().moe.ep_num_redundant_experts
|
||||
)
|
||||
@@ -450,8 +448,6 @@ class ExaoneMoEDecoderLayer(nn.Module):
|
||||
attention_bias = getattr(config, "attention_bias", False) or getattr(
|
||||
config, "bias", False
|
||||
)
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.self_attn = ExaoneMoEAttention(
|
||||
config=config,
|
||||
|
||||
@@ -45,7 +45,6 @@ class ExaoneMoEForCausalLMMTP(ExaoneMoEForCausalLM):
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
config.num_hidden_layers = 1
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
|
||||
@@ -224,7 +224,6 @@ class Gemma4MoE(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
self.hidden_size = hidden_size
|
||||
self.num_experts = config.num_experts
|
||||
self.tp_size = get_parallel().tp_size
|
||||
|
||||
# Per-expert output scale folded into routing weights so that
|
||||
# MoE's fused kernel computes: Σ_e (expert_e * w_e * scale_e)
|
||||
|
||||
@@ -598,7 +598,6 @@ class GigaChat35ForCausalLM(DeepseekV2WeightLoaderMixin, nn.Module):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.num_fused_shared_experts = 0
|
||||
|
||||
self.model = GigaChat35Model(
|
||||
|
||||
@@ -141,7 +141,6 @@ class GigaChat35ForCausalLMNextN(DeepseekV2WeightLoaderMixin, nn.Module):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.num_fused_shared_experts = 0
|
||||
self.draft_model_idx = draft_model_idx or 0
|
||||
|
||||
|
||||
@@ -258,7 +258,6 @@ class Glm4MoeAttention(nn.Module):
|
||||
self.rope_theta = rope_theta
|
||||
self.use_qk_norm = use_qk_norm
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size,
|
||||
@@ -431,7 +430,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
nn.Module.__init__(self)
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.moe_ep_size = get_parallel().moe_ep_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
self.num_fused_shared_experts = (
|
||||
@@ -1141,7 +1139,6 @@ class Glm4MoeForCausalLM(nn.Module):
|
||||
nn.Module.__init__(self)
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = 0
|
||||
self.determine_num_fused_shared_experts()
|
||||
|
||||
@@ -850,7 +850,6 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
super().__init__()
|
||||
config.moe_layer_freq = 1
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.determine_num_fused_shared_experts()
|
||||
|
||||
@@ -142,7 +142,6 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
if is_npu() and get_spec().speculative_draft_model_quantization is None:
|
||||
quant_config = None
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -123,7 +123,6 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
if is_npu() and get_spec().speculative_draft_model_quantization is None:
|
||||
quant_config = None
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -43,7 +43,6 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
|
||||
self.config = config
|
||||
self.use_data_parallel = get_mm().mm_enable_dp_encoder
|
||||
vision_utils.update_vit_attn_dummy_heads_config(self.config)
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = 0
|
||||
self.determine_num_fused_shared_experts()
|
||||
|
||||
@@ -391,7 +391,6 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
head_shard_size = get_parallel().attn_tp_size
|
||||
head_shard_rank = get_parallel().attn_tp_rank
|
||||
_head_shard_rank_getter = partial(getattr, get_parallel(), "attn_tp_rank")
|
||||
@@ -1237,7 +1236,6 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = text_config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.use_dsa = is_deepseek_dsa(text_config)
|
||||
self.num_fused_shared_experts = 0
|
||||
|
||||
@@ -137,7 +137,6 @@ class GlmOcrForConditionalGenerationNextN(GlmOcrForConditionalGeneration):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
_, rope_scaling = get_rope_config(config)
|
||||
self.is_mrope_enabled = "mrope_section" in (rope_scaling or {})
|
||||
|
||||
@@ -209,7 +209,6 @@ class GptOssSparseMoeBlock(nn.Module):
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.layer_id = layer_id
|
||||
self.hidden_size = config.hidden_size
|
||||
self.activation = config.hidden_act
|
||||
@@ -408,7 +407,6 @@ class GptOssAttention(nn.Module):
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.rope_theta = rope_theta
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size,
|
||||
@@ -563,9 +561,6 @@ class GptOssDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
# GptOss all layers are sparse
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
|
||||
@@ -118,7 +118,6 @@ class Grok1MoE(nn.Module):
|
||||
intermediate_size: int,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
reduce_results: bool = True,
|
||||
use_presharded_weights: bool = False,
|
||||
inplace: bool = True,
|
||||
|
||||
@@ -95,8 +95,6 @@ class HYV3MoEFused(nn.Module):
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().moe_tp_size
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.layer_id = layer_id
|
||||
self.alt_stream = alt_stream
|
||||
self.n_routed_experts = config.num_experts
|
||||
|
||||
@@ -323,7 +323,6 @@ class InternS2MobiusRoutedExpertBank(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.bank_id = bank_id
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.num_experts = config.num_experts
|
||||
self.topk = TopK(
|
||||
top_k=config.num_experts_per_tok,
|
||||
|
||||
@@ -303,9 +303,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
)
|
||||
# dispatcher
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
# TODO: we will support tp < ep in the future
|
||||
self.ep_size = get_parallel().tp_size
|
||||
|
||||
self.deepep_dispatcher = DeepEPDispatcher(
|
||||
group=get_parallel().tp_group.device_group,
|
||||
router_topk=self.top_k,
|
||||
@@ -432,7 +429,6 @@ class LLaDA2MoeAttention(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.total_kv_heads = config.num_key_value_heads
|
||||
self.dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
@@ -598,7 +594,6 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
hidden_size = config.hidden_size
|
||||
|
||||
self.input_layernorm = RMSNorm(hidden_size, eps=config.rms_norm_eps)
|
||||
self.dp_size = get_parallel().attn_dp_size
|
||||
self.attention = LLaDA2MoeAttention(
|
||||
config,
|
||||
layer_id,
|
||||
@@ -608,8 +603,6 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
self.layer_id = layer_id
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.is_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id)
|
||||
is_previous_layer_sparse = self._is_layer_sparse(config, layer_id=layer_id - 1)
|
||||
|
||||
@@ -98,7 +98,6 @@ class Llama4MoE(nn.Module):
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.device_module = torch.get_device_module()
|
||||
|
||||
@@ -385,8 +384,6 @@ class Llama4DecoderLayer(nn.Module):
|
||||
rope_theta = config.rope_parameters["rope_theta"]
|
||||
rope_scaling = config.rope_parameters
|
||||
max_position_embeddings = config.max_position_embeddings
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.self_attn = Llama4Attention(
|
||||
config=config,
|
||||
|
||||
@@ -428,9 +428,6 @@ class LongcatFlashDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.attn_boundary, self.moe_boundary = make_stages(
|
||||
(
|
||||
declare_attn(),
|
||||
@@ -676,7 +673,6 @@ class LongcatFlashForCausalLM(nn.Module):
|
||||
]
|
||||
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.model = LongcatFlashModel(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -73,7 +73,7 @@ from sglang.srt.model_loader.utils import should_deepgemm_weight_requant_ue8m0
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
from sglang.srt.models.longcat_flash import LongcatFlashForCausalLM, LongcatFlashMLP
|
||||
from sglang.srt.runtime_context import get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_stream
|
||||
from sglang.srt.utils import (
|
||||
BumpAllocator,
|
||||
add_prefix,
|
||||
@@ -162,9 +162,6 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.attn_boundary, self.ffn_boundary = make_stages(
|
||||
(declare_attn(), self.input_layernorm),
|
||||
(
|
||||
|
||||
@@ -187,8 +187,6 @@ class MellumAttention(Qwen3MoeAttention):
|
||||
self.rope_theta = rope_params.get("rope_theta", 10000.0)
|
||||
rope_scaling = rope_params if _get_rope_type(rope_params) != "default" else None
|
||||
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size,
|
||||
self.head_dim,
|
||||
@@ -398,9 +396,6 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
mlp_layer_types = cfg.mlp_layer_types
|
||||
num_experts = cfg.num_experts
|
||||
|
||||
@@ -530,7 +525,6 @@ class MellumForCausalLM(Qwen3MoeForCausalLM):
|
||||
self.capture_aux_hidden_states = False
|
||||
|
||||
self.attn_cp_size = get_parallel().attn_cp_size
|
||||
self.attn_cp_rank = get_parallel().attn_cp_rank
|
||||
self.moe_dp_size = get_parallel().moe_dp_size
|
||||
|
||||
assert self.attn_cp_size % self.moe_dp_size == 0, (
|
||||
|
||||
@@ -19,7 +19,6 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.qwen2 import Qwen2DecoderLayer
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class MiMoMultiTokenPredictorLayer(nn.Module):
|
||||
@@ -90,7 +89,6 @@ class MiMoMTP(nn.Module):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.model = MiMoMultiTokenPredictorLayer(
|
||||
|
||||
@@ -252,7 +252,6 @@ class MiMoV2MTP(MiMoV2ForCausalLM):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.model = MiMoV2ModelNextN(
|
||||
|
||||
@@ -163,7 +163,6 @@ class MultiHeadRMSNorm(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_world = get_parallel().attn_tp_size
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.num_heads = num_heads
|
||||
self.num_heads_per_tp = num_heads // self.tp_world
|
||||
self.head_dim = head_dim
|
||||
@@ -422,7 +421,6 @@ class MiniMaxM3MoE(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.top_k = config.num_experts_per_tok
|
||||
|
||||
@staticmethod
|
||||
@@ -564,7 +562,6 @@ class MiniMaxM3Attention(nn.Module):
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_size = attn_tp_size
|
||||
self.attn_tp_rank = attn_tp_rank
|
||||
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
|
||||
@@ -77,7 +77,6 @@ class MixtralMoE(nn.Module):
|
||||
layer_id: int,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -894,7 +894,6 @@ class MossVLSelfAttention(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
@@ -174,7 +174,6 @@ class NemotronHMoE(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.device_module = torch.get_device_module()
|
||||
|
||||
|
||||
@@ -64,7 +64,6 @@ class OlmoeMoE(nn.Module):
|
||||
intermediate_size: int,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
layer_id: int = 0,
|
||||
prefix: str = "",
|
||||
):
|
||||
|
||||
@@ -188,7 +188,6 @@ class PhiMoE(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.tp_size = get_parallel().tp_size
|
||||
|
||||
# Gate always runs at half / full precision for now.
|
||||
self.gate = ReplicatedLinear(
|
||||
|
||||
@@ -430,7 +430,6 @@ class Qwen2_5_VisionTransformer(nn.Module, RotaryPosMixin):
|
||||
)
|
||||
|
||||
# Resource prepared for vit cuda graph
|
||||
self.tp_size = 1 if use_data_parallel else get_parallel().tp_size
|
||||
self.max_context_len = max_context_len
|
||||
self.enable_cg = _is_cuda and envs.SGLANG_VIT_ENABLE_CUDA_GRAPH.get()
|
||||
|
||||
|
||||
@@ -394,8 +394,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
or get_moe_a2a_backend().is_deepep_v2()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
):
|
||||
# TODO: we will support tp < ep in the future
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.num_experts = (
|
||||
config.num_experts + get_exec().moe.ep_num_redundant_experts
|
||||
)
|
||||
@@ -1025,9 +1023,6 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
# Qwen2MoE all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
@@ -1117,8 +1112,6 @@ class Qwen2MoeModel(nn.Module):
|
||||
self.vocab_size = config.vocab_size
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
self.moe_dp_size = get_parallel().moe_dp_size
|
||||
|
||||
if self.pp_group.is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
|
||||
@@ -85,7 +85,6 @@ class Qwen3Attention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.start_layer = start_layer
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.total_num_heads = num_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
@@ -108,7 +107,6 @@ class Qwen3Attention(nn.Module):
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.rope_theta = rope_theta
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
norm_kwargs = (
|
||||
dict(
|
||||
|
||||
@@ -123,7 +123,6 @@ class Qwen3_5ForCausalLMMTP(nn.Module):
|
||||
quant_config = _mtp_quant_config(quant_config)
|
||||
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
|
||||
@@ -535,7 +535,6 @@ class Qwen3MoeAttention(nn.Module):
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.rope_theta = rope_theta
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size,
|
||||
@@ -838,9 +837,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
# Qwen3MoE all layers are sparse and have no nextn now
|
||||
self.is_layer_sparse = True
|
||||
is_previous_layer_sparse = True
|
||||
|
||||
@@ -45,7 +45,6 @@ class Qwen3MoeForCausalLMMTP(Qwen3MoeForCausalLM):
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
config.num_hidden_layers = 1
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM):
|
||||
# Deep-copy so MTP mutations below don't leak into the target's config.
|
||||
config = copy.deepcopy(config)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
if is_npu() and get_spec().speculative_draft_model_quantization is None:
|
||||
quant_config = None
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -47,7 +47,6 @@ class Qwen4ExpForCausalLMMTP(Qwen3_5ForCausalLMMTP):
|
||||
quant_config = _mtp_quant_config(quant_config)
|
||||
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
@@ -99,7 +99,6 @@ class SDARAttention(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.tp_size = get_parallel().tp_size
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
|
||||
@@ -117,7 +117,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
|
||||
|
||||
# Deepep / FuseEP support
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.num_experts = (
|
||||
config.num_experts + get_exec().moe.ep_num_redundant_experts
|
||||
)
|
||||
|
||||
@@ -192,9 +192,7 @@ class Step3TextAttention(nn.Module):
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.all_tp_rank = get_parallel().tp_rank
|
||||
self.total_num_heads = num_heads
|
||||
self.attn_tp_rank = attn_tp_rank
|
||||
self.layer_id = layer_id
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
self.num_heads = self.total_num_heads // attn_tp_size
|
||||
|
||||
@@ -64,8 +64,6 @@ class Step3p5MLP(nn.Module):
|
||||
swiglu_limit: Optional[float] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
tp_size: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
reduce_results: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -77,8 +75,6 @@ class Step3p5MLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -86,8 +82,6 @@ class Step3p5MLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
@@ -349,7 +343,6 @@ class Step3p5Attention(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.total_num_heads = num_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
@@ -372,7 +365,6 @@ class Step3p5Attention(nn.Module):
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.rope_theta = rope_theta
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.step3p5 import Step3p5DecoderLayer, Step3p5ForCausalLM
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -151,7 +150,6 @@ class Step3p5MTP(Step3p5ForCausalLM):
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.draft_model_idx = draft_model_idx
|
||||
|
||||
|
||||
@@ -969,7 +969,6 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
self.shared_experts = None
|
||||
|
||||
if get_moe_a2a_backend().is_deepep() or get_moe_a2a_backend().is_mori():
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.num_experts = (
|
||||
config.num_experts + get_exec().moe.ep_num_redundant_experts
|
||||
)
|
||||
@@ -1448,9 +1447,6 @@ class XllmDecoderLayer(nn.Module):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
# Determine if this layer is sparse (MoE) or dense
|
||||
mlp_only_layers = getattr(config, "mlp_only_layers", [])
|
||||
decoder_sparse_step = getattr(config, "decoder_sparse_step", 1)
|
||||
|
||||
@@ -100,7 +100,6 @@ class XverseMoE(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.rank = get_parallel().tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.n_routed_experts = config.num_experts
|
||||
self.top_k = config.moe_top_k
|
||||
|
||||
Reference in New Issue
Block a user