[Fix] Complete the deferred FFN all-reduce before a pipeline-parallel send (#41079)

This commit is contained in:
Cheng Wan
2026-09-24 12:13:57 -07:00
committed by GitHub
parent 6a14b80141
commit 928e683f94
23 changed files with 285 additions and 30 deletions
+11
View File
@@ -1116,6 +1116,17 @@ class FfnExit:
)
def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor:
"""Run the all-reduce a layer left for the next layer's input norm."""
if (
hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
and hidden_states._sglang_needs_allreduce_fusion
):
hidden_states = deferred_post_experts_all_reduce(hidden_states)
hidden_states._sglang_needs_allreduce_fusion = False
return hidden_states
@dataclass
class CommunicateContext:
process_group_sizes: Dict[ScatterMode, int]
+2
View File
@@ -37,6 +37,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -848,6 +849,7 @@ class BailingMoEModel(nn.Module):
),
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
@@ -18,7 +18,11 @@ from sglang.srt.distributed import (
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1008,6 +1012,7 @@ class BailingMoELinearModel(nn.Module):
zero_allocator=zero_allocator,
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
@@ -31,6 +31,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1392,6 +1393,7 @@ class BailingMoELinearModel(nn.Module):
dspark_aux_hidden_states.append(hidden_states + residual)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+2
View File
@@ -70,6 +70,7 @@ from sglang.srt.layers.aux_hidden_states import (
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
get_attn_tp_context,
)
@@ -3109,6 +3110,7 @@ class DeepseekV2Model(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
proxy_tensors = {
"hidden_states": hidden_states,
"residual": residual,
@@ -60,6 +60,7 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -1823,6 +1824,7 @@ class Dots3Model(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+2
View File
@@ -42,6 +42,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1138,6 +1139,7 @@ class Glm4MoeModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
@@ -38,6 +38,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
get_attn_tp_context,
)
@@ -837,6 +838,7 @@ class Glm4MoeLiteModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+2
View File
@@ -28,6 +28,7 @@ from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
get_attn_tp_context,
)
@@ -1215,6 +1216,7 @@ class Glm5NextModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
if self.config.mhc:
return PPProxyTensors({"hidden_states": hidden_states})
return PPProxyTensors(
+6 -1
View File
@@ -32,7 +32,11 @@ from sglang.srt.distributed import (
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -733,6 +737,7 @@ class GptOssModel(nn.Module):
else hidden_states
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+2
View File
@@ -25,6 +25,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -605,6 +606,7 @@ class LagunaModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+2
View File
@@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1105,6 +1106,7 @@ class MiMoV2Model(nn.Module):
hidden_states_before_norm = None
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+2
View File
@@ -37,6 +37,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
@@ -1186,6 +1187,7 @@ class MiniMaxM2Model(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+2
View File
@@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1539,6 +1540,7 @@ class MiniMaxM3Model(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+2
View File
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
Mamba2AttnBackend,
)
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.communicator import complete_deferred_allreduce
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
@@ -831,6 +832,7 @@ class NemotronHModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+2 -23
View File
@@ -34,8 +34,6 @@ from sglang.kernels.ops.elementwise.elementwise import (
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.distributed import (
get_pp_indices,
moe_expert_parallel_all_reduce,
moe_tensor_model_parallel_all_reduce,
tensor_model_parallel_all_reduce,
)
from sglang.srt.environ import envs
@@ -47,6 +45,7 @@ from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -61,7 +60,6 @@ from sglang.srt.layers.linear import (
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
)
@@ -1231,26 +1229,7 @@ class Qwen2MoeModel(nn.Module):
)
if not self.pp_group.is_last_rank:
if (
hidden_states is not None
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
and hidden_states._sglang_needs_allreduce_fusion
):
# The deferred reduction the next layer would have fused; no
# layer follows on this rank, so run it here. Unconditional --
# the skip flags that deferred it are what got us into this
# branch -- so it bypasses post_experts_all_reduce()'s guards
# while reusing its merge rule.
if can_merge_post_experts_all_reduce():
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
else:
if get_parallel().moe_ep_size > 1:
hidden_states = moe_expert_parallel_all_reduce(hidden_states)
if get_parallel().moe_tp_size > 1:
hidden_states = moe_tensor_model_parallel_all_reduce(
hidden_states
)
hidden_states._sglang_needs_allreduce_fusion = False
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+6 -1
View File
@@ -43,7 +43,11 @@ from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -1940,6 +1944,7 @@ class Qwen3_5ForCausalLM(nn.Module):
# Return intermediate tensors for pipeline parallelism
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+2
View File
@@ -24,6 +24,7 @@ import torch.nn as nn
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import complete_deferred_allreduce
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import get_layer_id
@@ -150,6 +151,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,
+2
View File
@@ -22,6 +22,7 @@ from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
enable_moe_dense_fully_dp,
)
from sglang.srt.layers.dp_attention import (
@@ -1171,6 +1172,7 @@ class SarvamMLAModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+6 -1
View File
@@ -16,7 +16,11 @@ from sglang.srt.distributed import (
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -487,6 +491,7 @@ class SDARMoeModel(nn.Module):
)
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
+6 -1
View File
@@ -11,7 +11,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
complete_deferred_allreduce,
)
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -735,6 +739,7 @@ class Step3p5Model(nn.Module):
)
# break
if not self.pp_group.is_last_rank:
hidden_states = complete_deferred_allreduce(hidden_states)
return PPProxyTensors(
{
"hidden_states": hidden_states,