[Fix] Stop counting a deferred FFN sum more than once: replicated TP1 shared expert, dense reduce_scatterv (#41195)

This commit is contained in:
Cheng Wan
2026-09-25 01:45:00 -07:00
committed by GitHub
parent 1409f469be
commit 402df23188
9 changed files with 318 additions and 6 deletions
+10 -1
View File
@@ -438,6 +438,7 @@ class LayerScatterModes:
mlp_mode: ScatterMode
middle_residual_mode: ScatterMode
layer_output_mode: ScatterMode
is_layer_sparse: bool = False
@classmethod
def init_new(cls, **kwargs):
@@ -448,6 +449,7 @@ class LayerScatterModes:
mlp_mode=cls._compute_mlp_mode(context),
middle_residual_mode=cls._compute_middle_residual_mode(context),
layer_output_mode=cls._compute_layer_output_mode(context),
is_layer_sparse=context.is_layer_sparse,
)
@classmethod
@@ -917,6 +919,7 @@ class LayerCommunicator:
forward_batch=forward_batch,
context=self._context,
allow_reduce_scatter=self.allow_reduce_scatter,
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
)
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
@@ -1617,6 +1620,7 @@ class CommunicateSummableTensorPairFn:
forward_batch: ForwardBatch,
context: CommunicateContext,
allow_reduce_scatter: bool = False,
is_layer_sparse: bool = False,
):
if get_parallel().tp_size == get_parallel().attn_dp_size:
group = get_parallel().tp_group
@@ -1626,7 +1630,12 @@ class CommunicateSummableTensorPairFn:
get_local_dp_buffer(group),
hidden_states,
)
if should_use_dp_reduce_scatterv():
# A MoE block leaves its sum to reduce_scatterv whenever it applies
# (should_skip_post_experts_all_reduce); a dense MLP does only under the
# published mlp_reduce_scatter, which needs allow_reduce_scatter.
if should_use_dp_reduce_scatterv() and (
allow_reduce_scatter or is_layer_sparse
):
get_parallel().tp_group.reduce_scatterv(
global_hidden_states,
output=hidden_states,
@@ -231,6 +231,7 @@ class DSACPCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn):
forward_batch: ForwardBatch,
context: CommunicateContext,
allow_reduce_scatter: bool = False,
**kwargs,
):
# for prefill: full -> attn tp scattered
# for decode: full -> attn tp full
+14
View File
@@ -779,6 +779,20 @@ def reduce_moe_output(hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states
def should_add_replicated_moe_output() -> bool:
"""Whether this rank adds an output every TP rank holds in full, such as a
shared expert replicated with tp_size=1, to its MoE output.
Call it after the MoE block's own reduction. When a later step still sums
the output over TP, only TP rank 0 adds it, so the sum counts it once.
"""
parallel = get_parallel()
summed_later = should_skip_post_experts_all_reduce(
is_tp_path=True
) and not post_experts_output_is_complete(is_tp_path=True)
return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0)
def can_merge_post_experts_all_reduce() -> bool:
"""Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce.
+7 -2
View File
@@ -115,6 +115,7 @@ from sglang.srt.layers.moe.utils import (
is_sbo_enabled,
is_shared_experts_fusion_disabled,
is_tbo_enabled,
should_add_replicated_moe_output,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8 import Fp8Config
@@ -1222,7 +1223,7 @@ class DeepseekV2MoE(nn.Module):
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated, so add them after all-reduce to
# avoid summing the same shared output once per TP rank.
if self._shared_expert_tp1:
if self._shared_expert_tp1 and should_add_replicated_moe_output():
final_hidden_states += shared_output
return final_hidden_states
@@ -1378,7 +1379,11 @@ class DeepseekV2MoE(nn.Module):
final_hidden_states = post_experts_all_reduce(final_hidden_states)
# TP1 shared experts are replicated, so add them after all-reduce to
# avoid summing the same shared output once per TP rank.
if shared_output is not None and self._shared_expert_tp1:
if (
shared_output is not None
and self._shared_expert_tp1
and should_add_replicated_moe_output()
):
final_hidden_states += shared_output
return final_hidden_states
+2 -1
View File
@@ -39,6 +39,7 @@ from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import should_add_replicated_moe_output
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -224,7 +225,7 @@ class LagunaMoE(nn.Module):
final = routed_out + shared_out
final = reduce_moe_output(final)
if self._shared_expert_tp1:
if self._shared_expert_tp1 and should_add_replicated_moe_output():
final = final + shared_out
return final
@@ -149,6 +149,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=True,
prefix=add_prefix(f"mlps", prefix),
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -35,6 +35,7 @@ def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes:
mlp_mode=mlp_mode,
middle_residual_mode=middle_residual_mode,
layer_output_mode=ScatterMode.TP_ATTN_FULL,
is_layer_sparse=is_sparse,
)