[Fix] Complete the deferred FFN all-reduce before a pipeline-parallel send (#41079)
This commit is contained in:
@@ -1116,6 +1116,17 @@ class FfnExit:
|
||||
)
|
||||
|
||||
|
||||
def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Run the all-reduce a layer left for the next layer's input norm."""
|
||||
if (
|
||||
hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
):
|
||||
hidden_states = deferred_post_experts_all_reduce(hidden_states)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
return hidden_states
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicateContext:
|
||||
process_group_sizes: Dict[ScatterMode, int]
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -848,6 +849,7 @@ class BailingMoEModel(nn.Module):
|
||||
),
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -18,7 +18,11 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1008,6 +1012,7 @@ class BailingMoELinearModel(nn.Module):
|
||||
zero_allocator=zero_allocator,
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1392,6 +1393,7 @@ class BailingMoELinearModel(nn.Module):
|
||||
dspark_aux_hidden_states.append(hidden_states + residual)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -70,6 +70,7 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
@@ -3109,6 +3110,7 @@ class DeepseekV2Model(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
proxy_tensors = {
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
|
||||
@@ -60,6 +60,7 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -1823,6 +1824,7 @@ class Dots3Model(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1138,6 +1139,7 @@ class Glm4MoeModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
@@ -837,6 +838,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -28,6 +28,7 @@ from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
get_attn_tp_context,
|
||||
)
|
||||
@@ -1215,6 +1216,7 @@ class Glm5NextModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
if self.config.mhc:
|
||||
return PPProxyTensors({"hidden_states": hidden_states})
|
||||
return PPProxyTensors(
|
||||
|
||||
@@ -32,7 +32,11 @@ from sglang.srt.distributed import (
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -733,6 +737,7 @@ class GptOssModel(nn.Module):
|
||||
else hidden_states
|
||||
)
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -605,6 +606,7 @@ class LagunaModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1105,6 +1106,7 @@ class MiMoV2Model(nn.Module):
|
||||
|
||||
hidden_states_before_norm = None
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
@@ -1186,6 +1187,7 @@ class MiniMaxM2Model(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1539,6 +1540,7 @@ class MiniMaxM3Model(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
Mamba2AttnBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.communicator import complete_deferred_allreduce
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
is_dp_attention_enabled,
|
||||
@@ -831,6 +832,7 @@ class NemotronHModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -34,8 +34,6 @@ from sglang.kernels.ops.elementwise.elementwise import (
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
moe_expert_parallel_all_reduce,
|
||||
moe_tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
@@ -47,6 +45,7 @@ from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -61,7 +60,6 @@ from sglang.srt.layers.linear import (
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
)
|
||||
@@ -1231,26 +1229,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
if (
|
||||
hidden_states is not None
|
||||
and hasattr(hidden_states, "_sglang_needs_allreduce_fusion")
|
||||
and hidden_states._sglang_needs_allreduce_fusion
|
||||
):
|
||||
# The deferred reduction the next layer would have fused; no
|
||||
# layer follows on this rank, so run it here. Unconditional --
|
||||
# the skip flags that deferred it are what got us into this
|
||||
# branch -- so it bypasses post_experts_all_reduce()'s guards
|
||||
# while reusing its merge rule.
|
||||
if can_merge_post_experts_all_reduce():
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
else:
|
||||
if get_parallel().moe_ep_size > 1:
|
||||
hidden_states = moe_expert_parallel_all_reduce(hidden_states)
|
||||
if get_parallel().moe_tp_size > 1:
|
||||
hidden_states = moe_tensor_model_parallel_all_reduce(
|
||||
hidden_states
|
||||
)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -43,7 +43,11 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -1940,6 +1944,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
|
||||
# Return intermediate tensors for pipeline parallelism
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -24,6 +24,7 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import complete_deferred_allreduce
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import get_layer_id
|
||||
@@ -150,6 +151,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
enable_moe_dense_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -1171,6 +1172,7 @@ class SarvamMLAModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -16,7 +16,11 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -487,6 +491,7 @@ class SDARMoeModel(nn.Module):
|
||||
)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
@@ -11,7 +11,11 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
complete_deferred_allreduce,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -735,6 +739,7 @@ class Step3p5Model(nn.Module):
|
||||
)
|
||||
# break
|
||||
if not self.pp_group.is_last_rank:
|
||||
hidden_states = complete_deferred_allreduce(hidden_states)
|
||||
return PPProxyTensors(
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user