[Fix] Stop counting a deferred FFN sum more than once: replicated TP1 shared expert, dense reduce_scatterv (#41195)
This commit is contained in:
@@ -438,6 +438,7 @@ class LayerScatterModes:
|
||||
mlp_mode: ScatterMode
|
||||
middle_residual_mode: ScatterMode
|
||||
layer_output_mode: ScatterMode
|
||||
is_layer_sparse: bool = False
|
||||
|
||||
@classmethod
|
||||
def init_new(cls, **kwargs):
|
||||
@@ -448,6 +449,7 @@ class LayerScatterModes:
|
||||
mlp_mode=cls._compute_mlp_mode(context),
|
||||
middle_residual_mode=cls._compute_middle_residual_mode(context),
|
||||
layer_output_mode=cls._compute_layer_output_mode(context),
|
||||
is_layer_sparse=context.is_layer_sparse,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -917,6 +919,7 @@ class LayerCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
)
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
||||
@@ -1617,6 +1620,7 @@ class CommunicateSummableTensorPairFn:
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
allow_reduce_scatter: bool = False,
|
||||
is_layer_sparse: bool = False,
|
||||
):
|
||||
if get_parallel().tp_size == get_parallel().attn_dp_size:
|
||||
group = get_parallel().tp_group
|
||||
@@ -1626,7 +1630,12 @@ class CommunicateSummableTensorPairFn:
|
||||
get_local_dp_buffer(group),
|
||||
hidden_states,
|
||||
)
|
||||
if should_use_dp_reduce_scatterv():
|
||||
# A MoE block leaves its sum to reduce_scatterv whenever it applies
|
||||
# (should_skip_post_experts_all_reduce); a dense MLP does only under the
|
||||
# published mlp_reduce_scatter, which needs allow_reduce_scatter.
|
||||
if should_use_dp_reduce_scatterv() and (
|
||||
allow_reduce_scatter or is_layer_sparse
|
||||
):
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
global_hidden_states,
|
||||
output=hidden_states,
|
||||
|
||||
@@ -231,6 +231,7 @@ class DSACPCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn):
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
allow_reduce_scatter: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
# for prefill: full -> attn tp scattered
|
||||
# for decode: full -> attn tp full
|
||||
|
||||
@@ -779,6 +779,20 @@ def reduce_moe_output(hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states
|
||||
|
||||
|
||||
def should_add_replicated_moe_output() -> bool:
|
||||
"""Whether this rank adds an output every TP rank holds in full, such as a
|
||||
shared expert replicated with tp_size=1, to its MoE output.
|
||||
|
||||
Call it after the MoE block's own reduction. When a later step still sums
|
||||
the output over TP, only TP rank 0 adds it, so the sum counts it once.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
summed_later = should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True
|
||||
) and not post_experts_output_is_complete(is_tp_path=True)
|
||||
return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0)
|
||||
|
||||
|
||||
def can_merge_post_experts_all_reduce() -> bool:
|
||||
"""Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce.
|
||||
|
||||
|
||||
@@ -115,6 +115,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
is_sbo_enabled,
|
||||
is_shared_experts_fusion_disabled,
|
||||
is_tbo_enabled,
|
||||
should_add_replicated_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config
|
||||
@@ -1222,7 +1223,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated, so add them after all-reduce to
|
||||
# avoid summing the same shared output once per TP rank.
|
||||
if self._shared_expert_tp1:
|
||||
if self._shared_expert_tp1 and should_add_replicated_moe_output():
|
||||
final_hidden_states += shared_output
|
||||
return final_hidden_states
|
||||
|
||||
@@ -1378,7 +1379,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
# TP1 shared experts are replicated, so add them after all-reduce to
|
||||
# avoid summing the same shared output once per TP rank.
|
||||
if shared_output is not None and self._shared_expert_tp1:
|
||||
if (
|
||||
shared_output is not None
|
||||
and self._shared_expert_tp1
|
||||
and should_add_replicated_moe_output()
|
||||
):
|
||||
final_hidden_states += shared_output
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
@@ -224,7 +225,7 @@ class LagunaMoE(nn.Module):
|
||||
final = routed_out + shared_out
|
||||
|
||||
final = reduce_moe_output(final)
|
||||
if self._shared_expert_tp1:
|
||||
if self._shared_expert_tp1 and should_add_replicated_moe_output():
|
||||
final = final + shared_out
|
||||
return final
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
prefix=add_prefix(f"mlps", prefix),
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -35,6 +35,7 @@ def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes:
|
||||
mlp_mode=mlp_mode,
|
||||
middle_residual_mode=middle_residual_mode,
|
||||
layer_output_mode=ScatterMode.TP_ATTN_FULL,
|
||||
is_layer_sparse=is_sparse,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user