diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 220e1b1b29..10d0c24fd9 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import ( CommunicateContext, CommunicateSummableTensorPairFn, ScatterMode, + reduce_output, ) from sglang.srt.layers.moe import ( get_deepep_mode, @@ -966,6 +967,7 @@ def _model_forward_tbo( input_data_scatter_mode: ScatterMode, layer_input_scatter_mode: ScatterMode, ): + inputs["hidden_states"] = reduce_output(inputs["hidden_states"]) inputs_arr = _model_forward_tbo_split_inputs( **inputs, input_data_scatter_mode=input_data_scatter_mode, diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index c507fe72a5..832904d3a4 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -18,6 +18,7 @@ from enum import Enum, auto from functools import cached_property, partial from typing import Callable, Dict, Optional, Tuple, Union +import msgspec import torch from sglang.srt.distributed import ( @@ -66,6 +67,7 @@ from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, deferred_post_experts_all_reduce, get_moe_a2a_backend, + post_experts_output_is_complete, should_use_dp_reduce_scatterv, should_use_flashinfer_cutlass_moe_fp4_allgather, ) @@ -82,6 +84,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo from sglang.srt.runtime_context import ( get_exec, get_forward, + get_lora, get_parallel, get_platform, get_spec, @@ -621,6 +624,68 @@ def _attn_input_update_and_read_residual(quant_format: str): return _update_and_read_residual_plain +class UnreducedOutput(msgspec.Struct, frozen=True): + """A layer output that still owes its FFN all-reduce, left for the next + layer's input norm. Hand it to the next layer, or pass it through + reduce_output() before reading it any other way.""" + + partial: torch.Tensor + + +def reduce_output( + hidden_states: Union[torch.Tensor, UnreducedOutput, None], +) -> Optional[torch.Tensor]: + """Run the all-reduce an UnreducedOutput still owes; pass anything else through.""" + if isinstance(hidden_states, UnreducedOutput): + return deferred_post_experts_all_reduce(hidden_states.partial) + return hidden_states + + +def layer_input_buffer( + hidden_states: Union[torch.Tensor, UnreducedOutput], +) -> torch.Tensor: + """The tensor holding a layer's input, for reusing its memory without reading it.""" + if isinstance(hidden_states, UnreducedOutput): + return hidden_states.partial + return hidden_states + + +def _batch_size(forward_batch: ForwardBatch) -> int: + return ( + forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0 + ) + + +def _deferred_reduction_runs_on_the_tp_group() -> bool: + """Whether deferred_post_experts_all_reduce reduces over the TP group object + itself, the group a dense FFN and reduce_moe_output reduce over.""" + parallel = get_parallel() + if parallel.moe_ep_size > 1 and parallel.moe_tp_size > 1: + # Some MoE blocks reduce EP and MoE-TP in two steps instead of merging. + return False + group = parallel.moe_ep_group if parallel.moe_ep_size > 1 else parallel.moe_tp_group + return group is parallel.tp_group + + +def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: + """Whether deferred_post_experts_all_reduce is the all-reduce the FFN would + have run itself: one full-precision sum over the same TP group, on an output + that is a plain partial sum, with no attention DP layout to restore. The + output is not a plain partial sum when the MoE combine already summed it, a + replicated shared expert is added after the reduction, or LoRA-B runs on + the unreduced activations.""" + return ( + _batch_size(forward_batch) > 0 + and not is_dp_attention_enabled() + and get_moe_a2a_backend().is_none() + and not post_experts_output_is_complete(is_tp_path=True) + and not get_exec().comm.enable_quant_communications + and not envs.SGLANG_SHARED_EXPERT_TP1.get() + and not get_lora().enable_lora + and _deferred_reduction_runs_on_the_tp_group() + ) + + class LayerCommunicator: def __init__( self, @@ -762,12 +827,15 @@ class LayerCommunicator: def prepare_attn( self, - hidden_states: torch.Tensor, + hidden_states: Union[torch.Tensor, UnreducedOutput], residual: torch.Tensor, forward_batch: ForwardBatch, quant_format: str = "", post_residual_addition: Optional[torch.Tensor] = None, ): + pending = isinstance(hidden_states, UnreducedOutput) + if pending: + hidden_states = hidden_states.partial # residual is None marks the first decoder layer, where the SP region # opens: re-evaluated per forward so a crash mid-loop cannot leak into # the next one. @@ -793,14 +861,21 @@ class LayerCommunicator: ) if hidden_states.shape[0] == 0: residual = hidden_states - elif ( - residual is not None - and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") - and hidden_states._sglang_needs_allreduce_fusion - ): - hidden_states, residual = self._reduce_output_and_update_and_read_residual( - hidden_states, residual, forward_batch + elif residual is not None and pending: + fused = ( + self._reduce_output_and_update_and_read_residual( + hidden_states, residual, forward_batch + ) + if post_residual_addition is None + else None ) + if fused is not None: + hidden_states, residual = fused + else: + hidden_states = deferred_post_experts_all_reduce(hidden_states) + hidden_states, residual = _attn_input_update_and_read_residual( + quant_format + )(self.input_layernorm, hidden_states, residual, post_residual_addition) else: hidden_states, residual = _attn_input_update_and_read_residual( quant_format @@ -831,12 +906,10 @@ class LayerCommunicator: hidden_states: torch.Tensor, residual: torch.Tensor, forward_batch: ForwardBatch, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: """Complete the sum the previous layer left, add it to the residual and - apply the input norm: - fused into one kernel when it takes this batch, else as an all-reduce - followed by the norm. Neither applies ``quant_format`` or - ``post_residual_addition``.""" + apply the input norm in one fused kernel; None when the kernel does not + take this batch. The result is not quantized for ``quant_format``.""" if ( apply_aiter_all_reduce_fusion(hidden_states, forward_batch) or apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) @@ -864,8 +937,7 @@ class LayerCommunicator: return self.input_layernorm.forward_with_allreduce_fusion( hidden_states, residual, use_attn_tp_group=False ) - hidden_states = deferred_post_experts_all_reduce(hidden_states) - return self.input_layernorm(hidden_states, residual) + return None def _tp_reduce_scatter( self, @@ -929,14 +1001,14 @@ class LayerCommunicator: def finish_layer_stack( self, - hidden_states: torch.Tensor, + hidden_states: Union[torch.Tensor, UnreducedOutput], residual: Optional[torch.Tensor], forward_batch: ForwardBatch, ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: """Complete what this layer left for a next layer. Call it on the last layer of this rank before its output reaches the final norm, the next pipeline rank, or any other consumer outside the layers.""" - return complete_deferred_allreduce(hidden_states), residual + return reduce_output(hidden_states), residual def should_use_reduce_scatter(self, forward_batch: ForwardBatch): if not self.allow_reduce_scatter: @@ -967,10 +1039,7 @@ class LayerCommunicator: """Whether the MoE may hand an unfinalized output to the next layer.""" return False - # NOTE: This function will cause torch recompilation - def should_fuse_mlp_allreduce_with_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: + def _ffn_sum_can_move_to_next_layer(self) -> bool: # When MOE_FULL is active (moe_cp allgather), fusion must be disabled because # the fusion path skips postprocess_layer which contains the moe_cp scatter. # Without scatter, hidden_states remain at MOE_FULL size while residual is at @@ -1003,17 +1072,17 @@ class LayerCommunicator: if get_attn_tp_context().input_scattered: return False - batch_size = ( - forward_batch.input_ids.shape[0] - if hasattr(forward_batch, "input_ids") - else 0 - ) - # When mlp_mode is SCATTERED, the MLP runs on scattered data with no TP # all-reduce, so there is nothing to fuse with the next layer. - if self.layer_scatter_modes.mlp_mode == ScatterMode.SCATTERED: - return False + return self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED + # NOTE: This function will cause torch recompilation + def should_fuse_mlp_allreduce_with_next_layer( + self, forward_batch: ForwardBatch + ) -> bool: + if not self._ffn_sum_can_move_to_next_layer(): + return False + batch_size = _batch_size(forward_batch) return ( ( apply_flashinfer_allreduce_fusion(batch_size) @@ -1030,6 +1099,20 @@ class LayerCommunicator: and (self._context.tp_size > 1) ) + def should_defer_ffn_reduction(self, forward_batch: ForwardBatch) -> bool: + """Whether the FFN leaves its output's all-reduce to the next layer's + input norm: whenever the fused kernel takes it, and otherwise when the + next layer would run the same all-reduce the FFN itself would have.""" + if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch): + return True + return ( + self._context.tp_size > 1 + and not self.is_last_layer + and self._ffn_sum_can_move_to_next_layer() + and _unfused_completion_matches_the_ffn(forward_batch) + and not self.should_use_reduce_scatter(forward_batch) + ) + # MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on. _SCATTER_MODE_SHARDED_AXES = { @@ -1079,7 +1162,7 @@ class FfnExit: # Deferring implies fusing: a handoff skips the post-experts all-reduce. self.fuse_mlp_allreduce = ( self.defer_moe_finalize - or communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) + or communicator.should_defer_ffn_reduction(forward_batch) ) self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch) self._scope = get_forward().scoped( @@ -1097,31 +1180,19 @@ class FfnExit: def finish( self, hidden_states: torch.Tensor, residual: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: """Leave the reduction to the next layer's input norm, or postprocess.""" if not isinstance(hidden_states, torch.Tensor): # A deferred MoE finalize handoff, consumed by the next prepare_attn. assert self.defer_moe_finalize, "unrequested deferred MoE handoff" return hidden_states, residual if self.fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True - return hidden_states, residual + return UnreducedOutput(hidden_states), residual return self.communicator.postprocess_layer( hidden_states, residual, self.forward_batch ) -def complete_deferred_allreduce(hidden_states: torch.Tensor) -> torch.Tensor: - """Run the all-reduce a layer left for the next layer's input norm.""" - if ( - hasattr(hidden_states, "_sglang_needs_allreduce_fusion") - and hidden_states._sglang_needs_allreduce_fusion - ): - hidden_states = deferred_post_experts_all_reduce(hidden_states) - hidden_states._sglang_needs_allreduce_fusion = False - return hidden_states - - @dataclass class CommunicateContext: process_group_sizes: Dict[ScatterMode, int] diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index d48f434586..178ef50f33 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -558,6 +558,9 @@ class MHCLayerCommunicator(LayerCommunicator): def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): return False + def should_defer_ffn_reduction(self, forward_batch): + return False + def should_use_reduce_scatter(self, forward_batch: ForwardBatch): if not self.allow_reduce_scatter: return False diff --git a/python/sglang/srt/layers/moe/__init__.py b/python/sglang/srt/layers/moe/__init__.py index 52ea18338e..9fcb7c6c95 100644 --- a/python/sglang/srt/layers/moe/__init__.py +++ b/python/sglang/srt/layers/moe/__init__.py @@ -14,6 +14,7 @@ from sglang.srt.layers.moe.utils import ( is_moe_input_scattered_across_dp_ranks, is_tbo_enabled, post_experts_all_reduce, + post_experts_output_is_complete, reduce_moe_output, should_skip_mlp_all_reduce, should_skip_post_experts_all_reduce, @@ -35,6 +36,7 @@ __all__ = [ "can_merge_post_experts_all_reduce", "deferred_post_experts_all_reduce", "post_experts_all_reduce", + "post_experts_output_is_complete", "reduce_moe_output", "should_skip_post_experts_all_reduce", "should_use_dp_reduce_scatterv", diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index 9b7ab4e08c..b3cd34f919 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -18,6 +18,7 @@ from sglang.srt.layers.communicator import ( CommunicateWithAllReduceAndLayerNormFn, LayerCommunicator, ScatterMode, + UnreducedOutput, get_attn_tp_context, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -221,17 +222,16 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): if ( residual is not None - and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") - and hidden_states._sglang_needs_allreduce_fusion + and isinstance(hidden_states, UnreducedOutput) and self._can_consume_post_moe_all_reduce( - forward_batch, int(hidden_states.shape[0]) + forward_batch, int(hidden_states.partial.shape[0]) ) ): if post_residual_addition is not None: residual = residual + post_residual_addition assert self.fusion_service is not None hidden_states, residual = self.fusion_service.all_reduce_residual_rms_norm( - local_contribution=hidden_states, + local_contribution=hidden_states.partial, residual=residual, gamma=_fused_norm_gamma(self.input_layernorm), ) diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 1c22f82353..c7d12ecb6c 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -34,8 +34,8 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, + reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -828,7 +828,7 @@ class BailingMoEModel(nn.Module): for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): if i in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_states.append( hidden_states if residual is None else hidden_states + residual ) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 87a1520127..4d42bc53c7 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -31,8 +31,9 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, + layer_input_buffer, + reduce_output, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm @@ -1385,9 +1386,9 @@ class BailingMoELinearModel(nn.Module): if ( capture_aux and i in self.layers_to_capture - and hidden_states.shape[0] != 0 + and layer_input_buffer(hidden_states).shape[0] != 0 ): - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) if residual is None: dspark_aux_hidden_states.append(hidden_states) else: diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 6b91e05853..0e8c1674d0 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -72,6 +72,7 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, enable_moe_dense_fully_dp, get_attn_tp_context, + layer_input_buffer, ) from sglang.srt.layers.communicator_dsa_cp import ( DSACPLayerCommunicator, @@ -2688,7 +2689,7 @@ class DeepseekV2DecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, next_full_attention_layer_id: Optional[int] = None, ) -> torch.Tensor: - hidden_states_orig = hidden_states + hidden_states_orig = layer_input_buffer(hidden_states) hidden_states, residual = ( self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( hidden_states, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 3c8ef85808..7b1e304d70 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -60,6 +60,7 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + UnreducedOutput, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1638,7 +1639,7 @@ class Dots3DecoderLayer(nn.Module): ) if should_allreduce_fusion: - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(hidden_states) if not should_allreduce_fusion: hidden_states, residual = self.layer_communicator.postprocess_layer( diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 1bb562fe62..e304818513 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -41,8 +41,8 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, + reduce_output, ) from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, @@ -1109,7 +1109,7 @@ class Glm4MoeModel(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): if i in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_states.append(hidden_states + residual) layer = self.layers[i] hidden_states, residual = layer( diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index f8423f568e..3e95bf2067 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -35,9 +35,9 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, get_attn_tp_context, + reduce_output, ) from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, @@ -804,7 +804,7 @@ class Glm4MoeLiteModel(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): if i in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_states.append(hidden_states + residual) layer = self.layers[i] hidden_states, residual = layer( diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index e1e29654e3..7a897e501a 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -28,9 +28,11 @@ from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, + UnreducedOutput, enable_moe_dense_fully_dp, get_attn_tp_context, + layer_input_buffer, + reduce_output, ) from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator from sglang.srt.layers.layernorm import RMSNorm @@ -923,7 +925,7 @@ class Glm5NextDecoderLayer(nn.Module): gemm_output_zero_allocator: BumpAllocator = None, prev_topk_indices: Optional[torch.Tensor] = None, ): - hidden_states_orig = hidden_states + hidden_states_orig = layer_input_buffer(hidden_states) hidden_states, residual = self.layer_communicator.prepare_attn( hidden_states, @@ -987,7 +989,7 @@ class Glm5NextDecoderLayer(nn.Module): ) if should_allreduce_fusion: - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(hidden_states) if not should_allreduce_fusion: hidden_states, residual = self.layer_communicator.postprocess_layer( @@ -1182,7 +1184,7 @@ class Glm5NextModel(nn.Module): ) with ctx: if i in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_state = self._prepare_aux_hidden_state( hidden_states, residual ) diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 4f668ddb5b..32859deb2c 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -32,7 +32,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, + reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -725,7 +725,7 @@ class GptOssModel(nn.Module): positions, hidden_states, forward_batch, residual ) if i + 1 in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_states.append( hidden_states + residual if residual is not None diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 84d06fc7f3..b9d875ef64 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -22,7 +22,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, + reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -592,7 +592,7 @@ class LagunaModel(nn.Module): aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_states.append( hidden_states + residual if residual is not None else hidden_states ) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 8bbbe5af88..7e66bf1ca5 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -39,6 +39,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, + UnreducedOutput, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1417,7 +1418,7 @@ class MiniMaxM3DecoderLayer(nn.Module): ) if should_allreduce_fusion: - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(hidden_states) else: hidden_states, residual = self.layer_communicator.postprocess_layer( hidden_states, residual, forward_batch diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index d20b290fb4..4f8ac807bb 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -25,15 +25,13 @@ from torch import nn from sglang.srt.compilation.compilation_config import register_split_op from sglang.srt.configs import NemotronHConfig from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.layers.activation import ReLU2 from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( HybridLinearAttnBackend, Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 +from sglang.srt.layers.communicator import UnreducedOutput, reduce_output from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, is_dp_attention_enabled, @@ -429,7 +427,7 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): ): hidden_states = self.mixer.forward(hidden_states) if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(hidden_states) else: hidden_states, residual = self.layer_communicator.postprocess_layer( hidden_states, residual, forward_batch @@ -533,7 +531,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): hidden_states, forward_batch, skip_reduce ) if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(hidden_states) return hidden_states, residual @@ -810,11 +808,7 @@ class NemotronHModel(nn.Module): aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - if residual is not None and getattr( - hidden_states, "_sglang_needs_allreduce_fusion", False - ): - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - hidden_states._sglang_needs_allreduce_fusion = False + hidden_states = reduce_output(hidden_states) aux_hidden_states.append( self._capture_hidden_states(hidden_states, residual, i) ) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 8a0f8b1288..3d66b475fb 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -46,7 +46,8 @@ from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_load from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, + UnreducedOutput, + reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -312,8 +313,7 @@ def _finish_mlp_output(hidden_states, *, expect_deferred: bool): if not expect_deferred: if isinstance(hidden_states, MoeFinalizeHandoff): raise RuntimeError("unexpected deferred-finalize handoff") - hidden_states._sglang_needs_allreduce_fusion = True - return hidden_states + return UnreducedOutput(hidden_states) if not isinstance(hidden_states, MoeFinalizeHandoff): raise RuntimeError("Qwen3.5 expected a FlashInfer deferred-finalize handoff") @@ -1938,7 +1938,7 @@ class Qwen3_5ForCausalLM(nn.Module): and layer_idx < 3 ): sep = self.hidden_size * layer_idx - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) hidden_states.add_( input_deepstack_embeds[:, sep : sep + self.hidden_size] ) diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 67278454ff..929dc5cbb0 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -24,7 +24,7 @@ import torch.nn as nn from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.layers.communicator import complete_deferred_allreduce +from sglang.srt.layers.communicator import reduce_output from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import get_layer_id @@ -107,7 +107,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): ): layer_idx += self.start_layer if layer_idx in self.layers_to_capture: - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) aux_hidden_states.append( hidden_states + residual if residual is not None else hidden_states ) @@ -140,7 +140,7 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): and layer_idx in self.deepstack_embed_to_decoder_layer ): sep = self.hidden_size * layer_idx - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) hidden_states.add_( input_deepstack_embeds[:, sep : sep + self.hidden_size] ) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index ba4688a931..ba776d043b 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -14,6 +14,7 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, + UnreducedOutput, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -641,7 +642,7 @@ class Step3p5DecoderLayer(nn.Module): fuse_mlp_allreduce = False if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(hidden_states) else: hidden_states, residual = self.layer_communicator.postprocess_layer( hidden_states, residual, forward_batch diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py new file mode 100644 index 0000000000..eff958b42a --- /dev/null +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -0,0 +1,70 @@ +"""The two-batch-overlap entry completes the sum its input still owes before it +splits the batch, so the split and the merge see the reduced tensor. CPU-only. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.batch_overlap import two_batch_overlap as tbo +from sglang.srt.layers import communicator +from sglang.srt.layers.communicator import ScatterMode, UnreducedOutput +from sglang.srt.utils import empty_context +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class TestTboEntryReducesItsInput(CustomTestCase): + def test_split_and_merge_see_the_reduced_tensor(self): + reduced = torch.full((3, 4), 2.0) + hidden_states = UnreducedOutput(torch.ones(3, 4)) + seen = {} + + def split(**kwargs): + seen["split"] = kwargs["hidden_states"] + return [{}, {}] + + def merge(output_a, output_b, original_len): + seen["original_len"] = original_len + return None, None + + with ( + patch.object( + communicator, "deferred_post_experts_all_reduce", lambda x: reduced + ), + patch.object(tbo, "_model_forward_tbo_split_inputs", split), + patch.object( + tbo, "execute_overlapped_operations", lambda **kwargs: [{}, {}] + ), + patch.object(tbo, "_model_forward_tbo_merge_outputs", merge), + patch.object( + tbo.deep_gemm_wrapper, + "configure_deep_gemm_num_sms", + lambda num_sms: empty_context(), + ), + ): + tbo._model_forward_tbo( + inputs=dict( + hidden_states=hidden_states, + residual=torch.zeros(3, 4), + positions=None, + forward_batch=None, + zero_allocator=None, + ), + operations_strategy=SimpleNamespace( + deep_gemm_num_sms=None, operations=[], tbo_delta_stages=0 + ), + input_data_scatter_mode=ScatterMode.TP_ATTN_FULL, + layer_input_scatter_mode=ScatterMode.TP_ATTN_FULL, + ) + + self.assertIs(seen["split"], reduced) + self.assertEqual(seen["original_len"], 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index 35e5fe04da..d41a2b0fdb 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -4,7 +4,7 @@ from unittest.mock import patch import pytest import torch -from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.layers.communicator import LayerCommunicator, UnreducedOutput from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, _retargeted_config, @@ -69,7 +69,7 @@ def eligible(): def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): """The last layer must fuse the reduction its predecessor skipped, or the - tagged tensor reaches the final norm unreduced; it must not skip its own.""" + unreduced output reaches the final norm; it must not skip its own.""" last = _communicator() last.successor_absorbs_all_reduce = False last.fusion_service = SimpleNamespace( @@ -78,8 +78,7 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): residual + 1, ) ) - hidden_states = torch.zeros(8, 8) - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = UnreducedOutput(torch.zeros(8, 8)) with ( patch.object( diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 16b7d17baa..39bba99f2a 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -5,7 +5,8 @@ import torch from sglang.srt.layers.communicator import ( LayerCommunicator, - complete_deferred_allreduce, + UnreducedOutput, + reduce_output, ) from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci @@ -19,9 +20,7 @@ def make_communicator(*, fuse, reduce_scatter, cls=LayerCommunicator): without the process-wide parallel state.""" communicator = cls.__new__(cls) if cls is LayerCommunicator: - communicator.should_fuse_mlp_allreduce_with_next_layer = MagicMock( - return_value=fuse - ) + communicator.should_defer_ffn_reduction = MagicMock(return_value=fuse) communicator.should_use_reduce_scatter = MagicMock(return_value=reduce_scatter) communicator.postprocess_layer = MagicMock( side_effect=lambda hidden_states, residual, forward_batch: ( @@ -49,11 +48,11 @@ class TestFfnExit(CustomTestCase): hidden_states = self.hidden_states * 2 return seen, ffn_exit.finish(hidden_states, self.residual) - def test_reduction_left_to_next_layer_is_marked(self): + def test_reduction_left_to_next_layer_is_unreduced(self): communicator = make_communicator(fuse=True, reduce_scatter=False) seen, (hidden_states, residual) = self.run_exit(communicator) self.assertEqual(seen, (True, False)) - self.assertTrue(hidden_states._sglang_needs_allreduce_fusion) + self.assertIsInstance(hidden_states, UnreducedOutput) self.assertIs(residual, self.residual) communicator.postprocess_layer.assert_not_called() @@ -65,9 +64,7 @@ class TestFfnExit(CustomTestCase): ) seen, (hidden_states, residual) = self.run_exit(communicator) self.assertEqual(seen, (False, reduce_scatter)) - self.assertFalse( - getattr(hidden_states, "_sglang_needs_allreduce_fusion", False) - ) + self.assertNotIsInstance(hidden_states, UnreducedOutput) communicator.postprocess_layer.assert_called_once() torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) self.assertIs(residual, self.residual) @@ -80,15 +77,12 @@ class TestFfnExit(CustomTestCase): communicator = make_communicator(fuse=fuse, reduce_scatter=False) with communicator.ffn_exit(self.forward_batch) as ffn_exit: seen = published_flags() - decide = communicator.should_fuse_mlp_allreduce_with_next_layer + decide = communicator.should_defer_ffn_reduction decide.return_value = not fuse hidden_states = self.hidden_states * 2 hidden_states, _ = ffn_exit.finish(hidden_states, self.residual) self.assertEqual(seen, (fuse, False)) - self.assertEqual( - getattr(hidden_states, "_sglang_needs_allreduce_fusion", False), - fuse, - ) + self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse) self.assertEqual(communicator.postprocess_layer.called, not fuse) def test_flags_are_restored_after_the_ffn(self): @@ -97,12 +91,12 @@ class TestFfnExit(CustomTestCase): self.assertEqual(published_flags(), before) def test_subclass_decisions_are_used(self): - class NeverFuses(LayerCommunicator): - def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): + class NeverDefers(LayerCommunicator): + def should_defer_ffn_reduction(self, forward_batch): return False communicator = make_communicator( - fuse=True, reduce_scatter=False, cls=NeverFuses + fuse=True, reduce_scatter=False, cls=NeverDefers ) seen, _ = self.run_exit(communicator) self.assertEqual(seen, (False, False)) @@ -132,30 +126,10 @@ class TestFfnExit(CustomTestCase): # The MoE may decline per forward; a tensor then takes the fused exit. hidden_states, _ = ffn_exit.finish(self.hidden_states * 2, self.residual) - self.assertTrue(hidden_states._sglang_needs_allreduce_fusion) - - def test_compiles_without_graph_breaks(self): - communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator.should_fuse_mlp_allreduce_with_next_layer = lambda forward_batch: ( - True - ) - communicator.should_use_reduce_scatter = lambda forward_batch: False - forward_batch = object() - - def layer(hidden_states, residual): - with communicator.ffn_exit(forward_batch) as ffn_exit: - if get_forward().fuse_mlp_allreduce: - hidden_states = hidden_states * 2 - return ffn_exit.finish(hidden_states, residual) - - torch._dynamo.reset() - compiled = torch.compile(layer, backend="eager", fullgraph=True) - hidden_states, residual = compiled(self.hidden_states, self.residual) - torch.testing.assert_close(hidden_states, self.hidden_states * 2) - self.assertIs(residual, self.residual) + self.assertIsInstance(hidden_states, UnreducedOutput) -class TestCompleteDeferredAllreduce(CustomTestCase): +class TestReduceOutput(CustomTestCase): def setUp(self): self.all_reduce = MagicMock(side_effect=lambda hidden_states: hidden_states * 3) patcher = patch( @@ -171,12 +145,12 @@ class TestCompleteDeferredAllreduce(CustomTestCase): hidden_states = torch.ones(3, 4) hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) - hidden_states = complete_deferred_allreduce(hidden_states) + hidden_states = reduce_output(hidden_states) self.all_reduce.assert_called_once() torch.testing.assert_close(hidden_states, torch.full((3, 4), 3.0)) - self.assertFalse(hidden_states._sglang_needs_allreduce_fusion) + self.assertNotIsInstance(hidden_states, UnreducedOutput) - complete_deferred_allreduce(hidden_states) + reduce_output(hidden_states) self.all_reduce.assert_called_once() def test_complete_hidden_states_pass_through(self): @@ -185,8 +159,8 @@ class TestCompleteDeferredAllreduce(CustomTestCase): hidden_states = torch.ones(3, 4) hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) - self.assertIs(complete_deferred_allreduce(hidden_states), hidden_states) - self.assertIsNone(complete_deferred_allreduce(None)) + self.assertIs(reduce_output(hidden_states), hidden_states) + self.assertIsNone(reduce_output(None)) self.all_reduce.assert_not_called() def test_finish_layer_stack_completes_the_last_layer(self): diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index ab00930d8f..d14d2654f6 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -21,12 +21,12 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL): - return types.SimpleNamespace( - _speculative_algo=None, - layer_scatter_modes=types.SimpleNamespace(mlp_mode=mlp_mode), - is_last_layer=False, - _context=types.SimpleNamespace(tp_size=4), - ) + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator._speculative_algo = None + communicator.layer_scatter_modes = types.SimpleNamespace(mlp_mode=mlp_mode) + communicator.is_last_layer = False + communicator._context = types.SimpleNamespace(tp_size=4) + return communicator @contextlib.contextmanager @@ -279,5 +279,132 @@ class TestFuseMlpAllReduceGate(CustomTestCase): ) +class TestDeferFfnReduction(CustomTestCase): + """The FFN leaves its sum to the next layer whenever the next layer runs the + all-reduce the FFN would have: fused when the kernel takes the batch, and + otherwise the same full-precision TP all-reduce.""" + + def _should_defer( + self, + *, + fused=False, + dp_attention=False, + a2a_none=True, + output_complete=False, + quant_communications=False, + reduce_scatter=False, + is_last_layer=False, + batch_size=8, + shared_expert_tp1=False, + lora=False, + tp_group=True, + ): + communicator = _fake_communicator() + communicator.is_last_layer = is_last_layer + communicator.should_use_reduce_scatter = lambda forward_batch: reduce_scatter + forward_batch = types.SimpleNamespace( + input_ids=types.SimpleNamespace(shape=(batch_size,)) + ) + with ( + patch.object(comm, "is_enable_moe_cp_allgather", return_value=False), + patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fused), + patch.object(comm, "_use_aiter", False), + patch.object( + comm, + "get_attn_tp_context", + return_value=types.SimpleNamespace(input_scattered=False), + ), + patch.object(comm, "is_dp_attention_enabled", return_value=dp_attention), + patch.object( + comm, + "get_moe_a2a_backend", + return_value=types.SimpleNamespace(is_none=lambda: a2a_none), + ), + patch.object( + comm, "post_experts_output_is_complete", return_value=output_complete + ), + patch.object( + comm, + "get_exec", + return_value=types.SimpleNamespace( + comm=types.SimpleNamespace( + enable_quant_communications=quant_communications + ) + ), + ), + patch.object( + comm.envs.SGLANG_SHARED_EXPERT_TP1, + "get", + return_value=shared_expert_tp1, + ), + patch.object( + comm, + "get_lora", + return_value=types.SimpleNamespace(enable_lora=lora), + ), + patch.object( + comm, "_deferred_reduction_runs_on_the_tp_group", return_value=tp_group + ), + get_parallel().override( + moe_ep_size=1, moe_tp_size=4, moe_dp_size=1, tp_size=4 + ), + ): + return communicator.should_defer_ffn_reduction(forward_batch) + + def test_defers_whether_or_not_the_fused_kernel_takes_the_batch(self): + self.assertTrue(self._should_defer(fused=True)) + self.assertTrue(self._should_defer(fused=False)) + + def test_keeps_the_reduction_when_the_next_layer_would_run_a_different_one( + self, + ): + for condition in ( + dict(dp_attention=True), + dict(a2a_none=False), + dict(output_complete=True), + dict(quant_communications=True), + dict(reduce_scatter=True), + dict(is_last_layer=True), + dict(batch_size=0), + dict(shared_expert_tp1=True), + dict(lora=True), + dict(tp_group=False), + ): + with self.subTest(**condition): + self.assertFalse(self._should_defer(**condition)) + + +class TestDeferredReductionGroup(CustomTestCase): + """The unfused completion runs on the same group object the FFN would have + reduced over, or the FFN keeps its reduction.""" + + def _runs_on_tp( + self, *, moe_ep_size, moe_tp_size, ep_is_tp=True, moe_tp_is_tp=True + ): + tp_group = object() + parallel = types.SimpleNamespace( + moe_ep_size=moe_ep_size, + moe_tp_size=moe_tp_size, + tp_group=tp_group, + moe_ep_group=tp_group if ep_is_tp else object(), + moe_tp_group=tp_group if moe_tp_is_tp else object(), + ) + with patch.object(comm, "get_parallel", return_value=parallel): + return comm._deferred_reduction_runs_on_the_tp_group() + + def test_pure_tp_and_pure_ep_reduce_over_the_tp_group(self): + self.assertTrue(self._runs_on_tp(moe_ep_size=1, moe_tp_size=4)) + self.assertTrue(self._runs_on_tp(moe_ep_size=4, moe_tp_size=1)) + + def test_a_separate_group_keeps_the_reduction_in_the_ffn(self): + self.assertFalse( + self._runs_on_tp(moe_ep_size=1, moe_tp_size=2, moe_tp_is_tp=False) + ) + self.assertFalse(self._runs_on_tp(moe_ep_size=4, moe_tp_size=1, ep_is_tp=False)) + + def test_hybrid_ep_tp_keeps_the_reduction_in_the_ffn(self): + self.assertFalse(self._runs_on_tp(moe_ep_size=2, moe_tp_size=2)) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 7f63aefdb4..e282d82b4c 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -136,14 +136,38 @@ class TestPrepareAttnSteps(CustomTestCase): platform(fusion=fusion) as (_, all_reduce), ): norm = Norm() - hidden_states = torch.ones(2, 4) - hidden_states._sglang_needs_allreduce_fusion = True + hidden_states = comm.UnreducedOutput(torch.ones(2, 4)) communicator(norm).prepare_attn(hidden_states, torch.zeros(2, 4), None) self.assertEqual(all_reduce.call_count, 0 if fusion else 1) self.assertEqual( norm.calls, ["fused_all_reduce_norm"] if fusion else ["norm"] ) + def test_a_pending_sum_keeps_the_quant_format_without_fusion(self): + with platform(use_aiter=True) as (kernels, all_reduce): + norm = Norm() + communicator(norm).prepare_attn( + comm.UnreducedOutput(torch.ones(2, 4)), + torch.zeros(2, 4), + None, + quant_format="fp8_per_token", + ) + self.assertEqual(all_reduce.call_count, 1) + self.assertTrue(kernels["_fused_rmsnorm_fp8_per_token_quant"].called) + self.assertEqual(norm.calls, []) + + def test_a_post_residual_addition_bypasses_the_fused_kernel(self): + with platform(fusion=True) as (_, all_reduce): + norm = Norm() + communicator(norm).prepare_attn( + comm.UnreducedOutput(torch.ones(2, 4)), + torch.zeros(2, 4), + None, + post_residual_addition=torch.ones(2, 4), + ) + self.assertEqual(all_reduce.call_count, 1) + self.assertEqual(norm.calls, ["norm"]) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 122527c5ec..7edea1bd82 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -45,6 +45,7 @@ class DeferringLayer(nn.Module): self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = ( lambda batch: defer ) + self.layer_communicator.should_defer_ffn_reduction = lambda batch: defer self.layer_communicator.should_use_reduce_scatter = lambda batch: False self.layer_communicator.postprocess_layer = lambda hidden, residual, batch: ( all_reduce(hidden), @@ -60,7 +61,7 @@ class DeferringLayer(nn.Module): *args, **kwargs, ): - hidden_states = comm.complete_deferred_allreduce(hidden_states) + hidden_states = comm.reduce_output(hidden_states) if residual is None: residual = hidden_states.clone() else: diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index a5ee8eccb6..36313747d8 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -9,7 +9,7 @@ from unittest.mock import patch import torch from torch import nn -from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.layers.communicator import LayerCommunicator, UnreducedOutput from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -19,7 +19,6 @@ TP_SIZE = 2 HIDDEN = 4 TOKENS = 3 NUM_LAYERS = 4 -MARKER = "_sglang_needs_allreduce_fusion" def all_reduce(hidden_states): @@ -41,14 +40,12 @@ class DeferringLayer(nn.Module): def forward( self, positions=None, hidden_states=None, forward_batch=None, residual=None, **_ ): - if getattr(hidden_states, MARKER, False): - hidden_states = all_reduce(hidden_states) + if isinstance(hidden_states, UnreducedOutput): + hidden_states = all_reduce(hidden_states.partial) residual = hidden_states if residual is None else hidden_states + residual if self.is_last_layer: return torch.ones_like(residual), residual - partial = torch.full_like(residual, 1 / TP_SIZE) - setattr(partial, MARKER, True) - return partial, residual + return UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE)), residual class SumNorm(nn.Module): diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py index acaeec9ae9..16ba9358b1 100644 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -16,9 +16,9 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" -MARKER = "_sglang_needs_allreduce_fusion" +UNREDUCED = "UnreducedOutput" EXIT = "finish_layer_stack" -COMPLETE = "complete_deferred_allreduce" +COMPLETE = "reduce_output" FINAL_NORMS = {"norm", "norm_f", "final_layernorm"} IN_PLACE = {"add_", "sub_", "mul_", "copy_"} @@ -60,20 +60,10 @@ def delegates_to_super(forward): def defers(node): """Asks whether to leave the reduction to the next layer (directly or - through ffn_exit), or sets the marker itself.""" - if any(calls(node, "ffn_exit")) or any( - calls(node, "should_fuse_mlp_allreduce_with_next_layer") - ): - return True + through ffn_exit), or wraps its output as unreduced itself.""" return any( - isinstance(sub, ast.Assign) - and isinstance(sub.value, ast.Constant) - and sub.value.value is True - and any( - isinstance(target, ast.Attribute) and target.attr == MARKER - for target in sub.targets - ) - for sub in ast.walk(node) + any(calls(node, name)) + for name in ("ffn_exit", "should_fuse_mlp_allreduce_with_next_layer", UNREDUCED) )