From f0ecf15c84d2b58d1726c2a1e87809ba0d71948f Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:51:48 -0700 Subject: [PATCH] [Refactor] Move CuTe DSL-fused layers onto the declared boundaries and FFN-exit kernel entries (#41431) --- python/sglang/srt/layers/communicator.py | 95 ++++++--- .../srt/layers/moe/cutedsl_ar_fusion.py | 160 +++++++++------ python/sglang/srt/models/deepseek_v2.py | 1 + python/sglang/srt/models/qwen2_moe.py | 1 + python/sglang/srt/models/qwen3_5.py | 7 +- .../unit/layers/moe/test_cutedsl_ar_fusion.py | 194 ++++++++++++++++-- .../unit/layers/test_communicator_ffn_exit.py | 25 ++- .../layers/test_declared_decoder_boundary.py | 17 +- 8 files changed, 375 insertions(+), 125 deletions(-) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index eca837e61c..c018f9b377 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -761,14 +761,28 @@ class UnreducedOutput(msgspec.Struct, frozen=True): reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None +class HandoffOutput(msgspec.Struct, frozen=True): + """A layer output that still owes work only its producer knows how to do (a + MoE's finalize and sum), left for the next layer's input. A fused kernel + there may do that work together with its own; anything else passes it + through reduce_output(), which calls ``complete()``.""" + + def complete(self) -> torch.Tensor: + """Do the owed work, unfused, and return the complete output.""" + raise NotImplementedError + + def reduce_output( - hidden_states: Union[torch.Tensor, UnreducedOutput, None], + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput, None], ) -> Optional[torch.Tensor]: - """Run the reduction an UnreducedOutput still owes; pass anything else through.""" + """Run the work an UnreducedOutput or a HandoffOutput still owes; pass + anything else through.""" if isinstance(hidden_states, UnreducedOutput): if hidden_states.reduce_and_redistribute is not None: return hidden_states.reduce_and_redistribute(hidden_states.partial) return hidden_states.group.all_reduce(hidden_states.partial) + if isinstance(hidden_states, HandoffOutput): + return hidden_states.complete() return hidden_states @@ -802,11 +816,10 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: class LayerCommunicator: - # Communicators built without __init__ (e.g. test doubles) publish no LoRA layout. + # Communicators built without __init__ (e.g. test doubles) publish no LoRA + # layout and try no fused kernel at the FFN exit. _publish_lora_layout: bool = False - # Whether this class's boundary steps may be chosen from both sides' - # declarations; a subclass that picks its own steps says no. - _takes_declared_boundaries = True + _ffn_exit_fusions: Tuple[Callable, ...] = () # A plain residual unless the layer is built with its own. _residual_ops: ResidualOps = ADD_AND_NORM @@ -847,6 +860,8 @@ class LayerCommunicator: ) # The fused kernels every batch's attention input tries first. self._attn_input_fusions = self._select_attn_input_fusions() + # The fused kernels of the next layer's input the FFN exit tries first. + self._ffn_exit_fusions = self._select_ffn_exit_fusions() self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) @@ -1004,8 +1019,7 @@ class LayerCommunicator: ) if not ( - self._takes_declared_boundaries - and (parallel.attn_cp_size == 1 or _cp_on_declarations()) + (parallel.attn_cp_size == 1 or _cp_on_declarations()) # MoE layers under attention DP and GQA prefill CP keep the # scatter-mode steps. and not ( @@ -1544,9 +1558,14 @@ class LayerCommunicator: mlp_reduce_scatter=mlp_reduce_scatter, complete=complete_now, ) - defer_moe_finalize = self.should_defer_moe_finalize(forward_batch) - # Deferring implies fusing: a handoff skips the post-experts all-reduce. - fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( + fusion = next( + filter(None, (fused(forward_batch) for fused in self._ffn_exit_fusions)), + None, + ) + defer_moe_finalize = fusion is FfnExitFusion.DEFER_MOE_FINALIZE + # A fused kernel that takes the sum, a handoff included, skips the + # post-experts all-reduce. + fuse_mlp_allreduce = fusion is not None or self._ffn_sum_moves_to_next_layer( forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step ) if fuse_mlp_allreduce: @@ -1629,13 +1648,19 @@ class LayerCommunicator: def finish_layer_stack( self, - hidden_states: Union[torch.Tensor, UnreducedOutput], + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], residual: Optional[torch.Tensor], forward_batch: ForwardBatch, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + *, + final_norm_takes_handoff: bool = False, + ) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]: """Complete what this layer left for a next layer. Call it on the last layer of this rank before its output reaches the final norm, the next - pipeline rank, or any other consumer outside the layers.""" + pipeline rank, or any other consumer outside the layers. A final norm + that does a producer's handoff together with its own work + (``final_norm_takes_handoff``) receives it as it is.""" + if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): + return hidden_states, residual return reduce_output(hidden_states), residual def should_use_reduce_scatter(self, forward_batch: ForwardBatch): @@ -1645,11 +1670,21 @@ class LayerCommunicator: forward_batch, self._postprocess_dp_step(forward_batch) ) - def should_defer_moe_finalize( - self, forward_batch: ForwardBatch, m: int | None = None - ) -> bool: - """Whether the MoE may hand an unfinalized output to the next layer.""" - return False + def _select_ffn_exit_fusions( + self, + ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: + """The fused kernels of the next layer's input that may take this layer's + FFN sum, in the order they are tried. Each returns what the exit does + when its kernel takes this batch, or None.""" + return (self._next_input_norm_takes_ffn_sum,) + + def _next_input_norm_takes_ffn_sum( + self, forward_batch: ForwardBatch + ) -> Optional["FfnExitFusion"]: + """The aiter / flashinfer AR + add + norm of the next layer's input.""" + if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch): + return FfnExitFusion.NEXT_INPUT + return None def _ffn_sum_can_move_to_next_layer(self, forward_batch: ForwardBatch) -> bool: # Under the MoE-CP all-gather the fusion path would skip postprocess_layer @@ -1712,10 +1747,8 @@ class LayerCommunicator: dp_step: Optional[Callable], ) -> bool: """Whether the FFN leaves its output's all-reduce to the next layer's - input norm: whenever the fused kernel takes it, and otherwise when the - next layer would run the same all-reduce the FFN itself would have.""" - if self.should_fuse_mlp_allreduce_with_next_layer(forward_batch): - return True + input when no fused kernel takes it: when the next layer would run the + same all-reduce the FFN itself would have.""" return ( self._context.tp_size > 1 and not self.is_last_layer @@ -1761,6 +1794,16 @@ def scatter_mode_layouts( } +class FfnExitFusion(Enum): + """What an FFN exit does when a fused kernel of the next layer's input takes + its sum.""" + + # The MoE hands its unfinalized output on; the next input finalizes it. + DEFER_MOE_FINALIZE = auto() + # The FFN leaves its all-reduce to the next input norm. + NEXT_INPUT = auto() + + class FfnCompletion(msgspec.Struct, frozen=True): """One FFN's reduction decision. The flags are published while the FFN runs; ``complete(hidden_states, residual)`` then either wraps the output for the @@ -2432,7 +2475,11 @@ def _attention_input_step( result = fused(owed, residual, forward_batch, post_residual_addition) if result is not None: return result - hidden_states = owed.partial + if isinstance(owed, HandoffOutput): + # No fused kernel took the handoff: its producer completes it. + hidden_states, owed = reduce_output(owed), None + else: + hidden_states = owed.partial if layer_input is not None: hidden_states, residual = layer_input(hidden_states, residual, context) if enters: diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index 886ea916fa..d22b530b25 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -10,14 +10,14 @@ from __future__ import annotations import logging from typing import Callable, Optional, Sequence -import msgspec import torch -from sglang.srt.layers.boundary_layout import SumGroup +from sglang.srt.layers.boundary_layout import SumGroup, TokenAxis from sglang.srt.layers.communicator import ( + FfnExitFusion, FusedMlpInput, + HandoffOutput, LayerCommunicator, - ScatterMode, UnreducedOutput, get_attn_tp_context, ) @@ -74,14 +74,20 @@ def _resolve_max_m(*, max_running_requests: int | None) -> int: return max(positive) -class MoeFinalizeHandoff(msgspec.Struct, frozen=True): - """Unfinalized routed output plus the separately gated shared contribution.""" +class MoeFinalizeHandoff(HandoffOutput, frozen=True): + """Unfinalized routed output plus the separately gated shared contribution. + The next layer's fused finalize + AR + add + norm takes it; ``finish`` is + the MoE's own unfused tail, for any other reader.""" routed_output: torch.Tensor expert_weights: torch.Tensor permuted_indices: torch.Tensor gated_shared_output: torch.Tensor m: int + finish: Callable[[], torch.Tensor] + + def complete(self) -> torch.Tensor: + return self.finish() @classmethod def from_flashinfer( @@ -90,7 +96,13 @@ class MoeFinalizeHandoff(msgspec.Struct, frozen=True): *, gated_shared_output: torch.Tensor, m: int, + reduce: Callable[[torch.Tensor], torch.Tensor], ) -> MoeFinalizeHandoff: + """``reduce`` is the MoE's own sum of its finalized output.""" + from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + finalize_flashinfer_trtllm_deferred_output, + ) + top_k = int(deferred_output.top_k) return cls( routed_output=deferred_output.gemm2_out.view( @@ -102,6 +114,11 @@ class MoeFinalizeHandoff(msgspec.Struct, frozen=True): )[:m], gated_shared_output=gated_shared_output, m=int(m), + finish=lambda: reduce( + finalize_flashinfer_trtllm_deferred_output( + deferred_output, gated_shared_output + ) + ), ) @@ -178,17 +195,34 @@ class CuteDSLFusionService: class CuteDSLFusionLayerCommunicator(LayerCommunicator): - # Chooses its own boundary steps, not from the declarations. - _takes_declared_boundaries = False - fusion_service: CuteDSLFusionService | None = None + # The FFN exit's CuteDSL kernels, which install() gives from what it knows + # of this layer and the one after it. + _cutedsl_exit_fusions: tuple = () - # The runner can defer and a successor or the final norm consumes the handoff. - may_defer_moe_finalize: bool = False - # False on the last layer: the final norm does not all-reduce. - successor_absorbs_all_reduce: bool = False - # A replicated output follows the reduction; moving it would scale that by tp. - owes_local_reduction: bool = False + def install( + self, + service: CuteDSLFusionService, + *, + hands_off_finalize: bool, + next_input_absorbs: bool, + output_is_replicated: bool, + ) -> None: + """Take the model's fusion service and choose the FFN exit's CuteDSL + kernels: the MoE may hand off its finalize when it can and something + after it takes the handoff (``hands_off_finalize``); the FFN may leave + its all-reduce to the next layer's AR + norm when that layer has one, + unless a replicated output follows the reduction (moving it would scale + that by tp).""" + self.fusion_service = service + self._cutedsl_exit_fusions = ( + (self._defer_moe_finalize_cutedsl,) if hands_off_finalize else () + ) + ( + (self._absorb_all_reduce_cutedsl,) + if next_input_absorbs and not output_is_replicated + else () + ) + self._ffn_exit_fusions = self._select_ffn_exit_fusions() def _select_attn_input_fusions(self): return ( @@ -201,19 +235,13 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): self, owed, residual, forward_batch, post_residual_addition ): """Finish a deferred MoE finalize with the all-reduce, residual add and - input norm.""" + input norm, when this layer's kernel takes the batch; otherwise the + handoff is completed by its producer's own tail.""" if not isinstance(owed, MoeFinalizeHandoff): return None - if not self._should_use_finalize(forward_batch, owed.m): - raise RuntimeError( - "received deferred MoE output on an ineligible path " - f"(M={owed.m}, mode={forward_batch.forward_mode})" - ) gamma = _fused_norm_gamma(self.input_layernorm) - if gamma is None: - raise RuntimeError( - "deferred MoE finalize requires a fusable RMSNorm flavour" - ) + if gamma is None or not self._should_use_finalize(forward_batch, owed.m): + return None if post_residual_addition is not None: residual = residual + post_residual_addition assert self.fusion_service is not None @@ -245,13 +273,14 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): fusions = super()._select_mlp_input_fusions() parallel = get_parallel() if ( - self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL + TokenAxis.ATTN_TP_SCATTER not in self.input_rows.sharded + # The workspace sums over TP, which is then the attention-TP group. and parallel.attn_tp_size == parallel.tp_size and _fused_norm_gamma(self.post_attention_layernorm) is not None ): return ( FusedMlpInput( - completes=SumGroup.TP, + completes=SumGroup.ATTN_TP, run=self._mlp_input_reduce_output_and_update_and_read_residual_cutedsl, may_return_new_residual=True, ), @@ -292,25 +321,29 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): and not get_exec().comm.enable_quant_communications ) - def _can_absorb_post_moe_all_reduce( - self, forward_batch: ForwardBatch, m: int - ) -> bool: - """Outgoing: skip our own all-reduce because the next layer absorbs it.""" - return ( - self.successor_absorbs_all_reduce - and not self.owes_local_reduction - and self._can_consume_post_moe_all_reduce(forward_batch, m) - ) + def _select_ffn_exit_fusions(self): + return (*self._cutedsl_exit_fusions, *super()._select_ffn_exit_fusions()) - def should_defer_moe_finalize( - self, forward_batch: ForwardBatch, m: int | None = None - ) -> bool: - """Deferring skips the post-experts all-reduce on the promise of a handoff.""" - if not self.may_defer_moe_finalize: - return False - if m is None: - m = int(forward_batch.input_ids.shape[0]) - return self._should_use_finalize(forward_batch, m) + def _defer_moe_finalize_cutedsl( + self, forward_batch: ForwardBatch + ) -> Optional[FfnExitFusion]: + """The MoE hands its unfinalized output to the next layer's finalize + + all-reduce + norm.""" + if self._should_use_finalize( + forward_batch, int(forward_batch.input_ids.shape[0]) + ): + return FfnExitFusion.DEFER_MOE_FINALIZE + return None + + def _absorb_all_reduce_cutedsl( + self, forward_batch: ForwardBatch + ) -> Optional[FfnExitFusion]: + """The next layer's AR + norm takes the post-experts all-reduce.""" + if self._can_consume_post_moe_all_reduce( + forward_batch, int(forward_batch.input_ids.shape[0]) + ): + return FfnExitFusion.NEXT_INPUT + return None def _common_eligible(self, forward_batch: ForwardBatch, m: int) -> bool: parallel = get_parallel() @@ -325,20 +358,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): and not get_attn_tp_context().input_scattered and get_moe_a2a_backend().is_none() and self._context.tp_size > 1 - # Restates the base's moe-cp, MOE_FULL and SCATTERED refusals. - and self.layer_scatter_modes.mlp_mode is ScatterMode.FULL + # The FFN runs on the full rows, not each rank's own slice. + and TokenAxis.ATTN_TP_SCATTER + not in self._batch_steps(forward_batch).ffn_input_rows.sharded ) - def should_fuse_mlp_allreduce_with_next_layer( - self, forward_batch: ForwardBatch - ) -> bool: - m = int(forward_batch.input_ids.shape[0]) - if self.should_defer_moe_finalize(forward_batch, m): - return True - if self._can_absorb_post_moe_all_reduce(forward_batch, m): - return True - return super().should_fuse_mlp_allreduce_with_next_layer(forward_batch) - def install_cutedsl_fusion( layers: Sequence[torch.nn.Module], @@ -387,6 +411,7 @@ def install_cutedsl_fusion( top_k=top_k, rms_epsilon=rms_epsilon, ) + hands_off = 0 for index, layer in enumerate(layers): communicator = layer.layer_communicator if not isinstance(communicator, CuteDSLFusionLayerCommunicator): @@ -398,15 +423,18 @@ def install_cutedsl_fusion( has_consumer = isinstance( successor.layer_communicator, CuteDSLFusionLayerCommunicator ) - communicator.fusion_service = service - communicator.may_defer_moe_finalize = ( - bool(can_defer_finalize(layer)) and has_consumer - ) - communicator.successor_absorbs_all_reduce = successor is not None and ( - isinstance(successor.layer_communicator, CuteDSLFusionLayerCommunicator) - ) - communicator.owes_local_reduction = ( - requires_local_reduction is not None and requires_local_reduction(layer) + hands_off_finalize = bool(can_defer_finalize(layer)) and has_consumer + hands_off += hands_off_finalize + communicator.install( + service, + hands_off_finalize=hands_off_finalize, + # False on the last layer: the final norm does not all-reduce. + next_input_absorbs=successor is not None + and isinstance( + successor.layer_communicator, CuteDSLFusionLayerCommunicator + ), + output_is_replicated=requires_local_reduction is not None + and bool(requires_local_reduction(layer)), ) logger.info( "Installed one %s FlashInfer MNNVL CuTe DSL fusion handle for %d of %d layers " @@ -414,7 +442,7 @@ def install_cutedsl_fusion( label, len(fusion_layers), len(layers), - sum(layer.layer_communicator.may_defer_moe_finalize for layer in fusion_layers), + hands_off, ) return service diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index b41e79e8b3..d869bead24 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -1113,6 +1113,7 @@ class DeepseekV2MoE(nn.Module): final_hidden_states, gated_shared_output=shared_output, m=hidden_states.shape[0], + reduce=post_experts_all_reduce, ) all_reduce_done = False diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 743d15a06f..f4875b8920 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -865,6 +865,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): final_hidden_states, gated_shared_output=shared_output, m=num_tokens, + reduce=reduce_moe_output, ) if shared_output is not None: diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index a144068857..30e2011fe2 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1856,7 +1856,12 @@ class Qwen3_5ForCausalLM(nn.Module): last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( - hidden_states, residual, forward_batch + hidden_states, + residual, + forward_batch, + # The final norm below finalizes a deferred MoE output in its kernel. + final_norm_takes_handoff=self.flashinfer_mnnvl_cutedsl_fusion is not None + and self.pp_group.is_last_rank, ) # Return intermediate tensors for pipeline parallelism diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index 92225791d4..cada5f5ba5 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -1,17 +1,18 @@ from functools import partial from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest import torch -from sglang.srt.layers.boundary_layout import SumGroup +from sglang.srt.layers.boundary_layout import Layout, SumGroup, TokenAxis from sglang.srt.layers.communicator import ( ADD_AND_NORM, + FfnExitFusion, LayerCommunicator, - ScatterMode, UnreducedOutput, _attention_input_step, + reduce_output, ) from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, @@ -93,12 +94,16 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): """The last layer must fuse the reduction its predecessor skipped, or the unreduced output reaches the final norm; it must not skip its own.""" last = _communicator() - last.successor_absorbs_all_reduce = False - last.fusion_service = SimpleNamespace( - all_reduce_residual_rms_norm=lambda *, local_contribution, residual, gamma: ( - local_contribution + 1, - residual + 1, - ) + last.install( + SimpleNamespace( + all_reduce_residual_rms_norm=lambda *, local_contribution, residual, gamma: ( + local_contribution + 1, + residual + 1, + ) + ), + hands_off_finalize=False, + next_input_absorbs=False, + output_is_replicated=False, ) hidden_states = UnreducedOutput(torch.zeros(8, 8)) @@ -119,7 +124,7 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): out_hidden, _ = last.prepare_attn(hidden_states, torch.zeros(8, 8), _DECODE) assert torch.equal(out_hidden, torch.ones(8, 8)) - assert last._can_absorb_post_moe_all_reduce(_DECODE, 8) is False + assert last._ffn_exit_fusions == (last._next_input_norm_takes_ffn_sum,) def test_cutedsl_entries_come_before_the_base_fused_kernel(): @@ -135,22 +140,67 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel(): f"{_MODULE}.get_parallel", return_value=SimpleNamespace(attn_tp_size=2, tp_size=2), ): - comm.layer_scatter_modes = SimpleNamespace( - layer_input_mode=ScatterMode.TP_ATTN_FULL - ) + comm._declared = SimpleNamespace(input_rows=Layout(frozenset())) fusions = comm._select_mlp_input_fusions() assert [f.run for f in fusions] == [cutedsl, base] - # The workspace reduces over the TP group, which is the attention-TP - # group here; both kernels hand back a new residual. - assert [f.completes for f in fusions] == [SumGroup.TP, SumGroup.ATTN_TP] + # The workspace reduces over the TP group, which is the attention-TP group here. + assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2 assert all(f.may_return_new_residual for f in fusions) - # A scattered residual is gathered first, which the workspace does not do. - comm.layer_scatter_modes = SimpleNamespace( - layer_input_mode=ScatterMode.SCATTERED + # A residual on each rank's slice is gathered first, which the + # workspace does not do. + comm._declared = SimpleNamespace( + input_rows=Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) ) assert [f.run for f in comm._select_mlp_input_fusions()] == [base] +def test_the_fusion_runs_only_on_the_ffn_full_rows(): + """The finalize and the AR + norm sum over the whole TP group, so they need + the batch's FFN input on the full rows, not each rank's own slice (a2a, the + fp4 all-gather, DWDP or a dense MLP on every rank).""" + comm = _communicator() + comm.fusion_service = SimpleNamespace(supports=lambda m: True) + comm._context = SimpleNamespace(tp_size=2) + with ( + patch( + f"{_MODULE}.get_parallel", + return_value=SimpleNamespace(attn_cp_size=1), + ), + patch(f"{_MODULE}.is_dp_attention_enabled", return_value=False), + patch( + f"{_MODULE}.get_attn_tp_context", + return_value=SimpleNamespace(input_scattered=False), + ), + patch( + f"{_MODULE}.get_moe_a2a_backend", + return_value=SimpleNamespace(is_none=lambda: True), + ), + ): + for sharded, eligible in ( + (frozenset(), True), + (frozenset({TokenAxis.ATTN_TP_SCATTER}), False), + ): + comm._batch_steps = lambda fb, rows=Layout(sharded): SimpleNamespace( + ffn_input_rows=rows + ) + assert comm._common_eligible(_DECODE, 8) is eligible + + +def test_the_exit_tries_the_handoff_then_the_absorb_then_the_base_kernel(): + comm = _communicator() + comm.install( + SimpleNamespace(), + hands_off_finalize=True, + next_input_absorbs=True, + output_is_replicated=False, + ) + assert comm._ffn_exit_fusions == ( + comm._defer_moe_finalize_cutedsl, + comm._absorb_all_reduce_cutedsl, + comm._next_input_norm_takes_ffn_sum, + ) + + def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): """A TP1 shared expert (or TP1 dense MLP) is not partial; handing its layer's reduction onward would scale the replicated output by tp_size.""" @@ -161,13 +211,19 @@ def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): _install(layers, requires_local_reduction=lambda layer: layer.replicated) replicated, plain = (layer.layer_communicator for layer in layers[:2]) + def exit_fusion(communicator): + return next( + filter(None, (fused(_DECODE) for fused in communicator._ffn_exit_fusions)), + None, + ) + with patch.object( LayerCommunicator, "should_fuse_mlp_allreduce_with_next_layer", return_value=False, ): - assert replicated.should_fuse_mlp_allreduce_with_next_layer(_DECODE) is False - assert plain.should_fuse_mlp_allreduce_with_next_layer(_DECODE) is True + assert exit_fusion(replicated) is None + assert exit_fusion(plain) is FfnExitFusion.NEXT_INPUT # Consuming what a predecessor skipped stays independently eligible. assert replicated._can_consume_post_moe_all_reduce(_DECODE, 8) is True @@ -251,6 +307,7 @@ def test_the_handoff_views_the_producer_storage(): ), gated_shared_output=torch.empty(m, 16, dtype=torch.bfloat16), m=m, + reduce=lambda h: h, ) assert handoff.routed_output.data_ptr() == gemm2_out.data_ptr() @@ -258,6 +315,101 @@ def test_the_handoff_views_the_producer_storage(): assert tuple(handoff.expert_weights.shape) == (m, top_k) +def _handoff(finish): + rows = torch.zeros(2, 8) + return MoeFinalizeHandoff( + routed_output=rows, + expert_weights=rows, + permuted_indices=rows, + gated_shared_output=rows, + m=2, + finish=finish, + ) + + +def test_its_producer_completes_a_handoff_for_any_other_reader(): + """Only the fused kernel reads the handoff's parts; everything else gets + the MoE's own unfused tail, run once.""" + finished = torch.full((2, 8), 3.0) + finish = MagicMock(return_value=finished) + assert reduce_output(_handoff(finish)) is finished + finish.assert_called_once_with() + + +def test_from_flashinfer_completes_with_the_moe_s_own_sum(): + deferred = SimpleNamespace( + gemm2_out=torch.zeros(4, 8), + expert_weights=torch.zeros(2, 2), + expanded_idx_to_permuted_idx=torch.zeros(2, 2, dtype=torch.int32), + top_k=2, + ) + shared = torch.ones(2, 8) + with patch( + "sglang.srt.layers.moe.moe_runner.flashinfer_trtllm." + "finalize_flashinfer_trtllm_deferred_output", + side_effect=lambda d, s: s + 1, + ) as finalize: + handoff = MoeFinalizeHandoff.from_flashinfer( + deferred, gated_shared_output=shared, m=2, reduce=lambda h: h * 10 + ) + finalize.assert_not_called() + torch.testing.assert_close(handoff.complete(), torch.full((2, 8), 20.0)) + finalize.assert_called_once_with(deferred, shared) + + +def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): + """The consumer asks only whether its own kernel takes the batch; when it + does not, the handoff is completed and the input read as usual.""" + + class AddNorm(torch.nn.Module): + def forward(self, x, residual=None, post_residual_addition=None): + residual.add_(x) + return residual * 2, residual + + comm = _communicator() + comm.input_layernorm = AddNorm() + comm.fusion_service = SimpleNamespace( + finalize=lambda **kw: pytest.fail("the kernel does not take this batch") + ) + finish = MagicMock(return_value=torch.ones(2, 8)) + with ( + # A norm the kernel could fold in, on a batch it does not take. + patch(f"{_MODULE}._fused_norm_gamma", return_value=torch.ones(8)), + patch.object( + CuteDSLFusionLayerCommunicator, "_should_use_finalize", return_value=False + ), + patch.object( + CuteDSLFusionLayerCommunicator, "_common_eligible", return_value=False + ), + patch.object( + CuteDSLFusionLayerCommunicator, + "_finish_prepare_attn", + lambda self, *, hidden_states, residual, forward_batch: ( + hidden_states, + residual, + ), + ), + ): + hidden, residual = comm.prepare_attn( + _handoff(finish), torch.full((2, 8), 2.0), _DECODE + ) + finish.assert_called_once_with() + torch.testing.assert_close(residual, torch.full((2, 8), 3.0)) + torch.testing.assert_close(hidden, torch.full((2, 8), 6.0)) + + +def test_the_layer_stack_hands_a_handoff_only_to_a_final_norm_that_takes_it(): + comm = LayerCommunicator.__new__(LayerCommunicator) + for takes, completed in ((True, 0), (False, 1)): + finish = MagicMock(return_value=torch.ones(2, 8)) + handoff = _handoff(finish) + hidden, _ = comm.finish_layer_stack( + handoff, torch.zeros(2, 8), _DECODE, final_norm_takes_handoff=takes + ) + assert finish.call_count == completed + assert (hidden is handoff) == takes + + def test_early_shared_load_touches_only_the_finalize_routes(): """Only a fused finalize has a completed shared-expert handoff to load early; the standalone all-reduce must keep the safe ordering.""" diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 71b14b11de..f75f2f964b 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -92,8 +92,10 @@ def make_communicator( if cls is LayerCommunicator: communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse) else: - # A subclass decides through its own hooks; no TP left to defer over. + # A subclass decides through its own fused kernels; no TP left to defer + # over otherwise. communicator._context = types.SimpleNamespace(tp_size=1) + communicator._ffn_exit_fusions = communicator._select_ffn_exit_fusions() communicator._ffn_leaves_sum_to_reduce_scatter = MagicMock( return_value=reduce_scatter ) @@ -219,6 +221,23 @@ class TestFfnExit(CustomTestCase): self.run_exit(make_communicator(fuse=True, reduce_scatter=True)) self.assertEqual(published_flags(), before) + def test_the_next_input_s_fused_kernel_is_tried_first(self): + """The base exit asks the next layer's AR + add + norm before the + unfused condition, and leaves the sum to it when it takes the batch.""" + for takes in (False, True): + with self.subTest(takes=takes): + communicator = make_communicator(fuse=False, reduce_scatter=False) + communicator.should_fuse_mlp_allreduce_with_next_layer = MagicMock( + return_value=takes + ) + communicator._ffn_exit_fusions = communicator._select_ffn_exit_fusions() + seen, _ = self.run_exit(communicator) + self.assertEqual(seen, (takes, False)) + communicator.should_fuse_mlp_allreduce_with_next_layer.assert_called_once() + self.assertEqual( + communicator._ffn_sum_moves_to_next_layer.called, not takes + ) + def test_subclass_decisions_are_used(self): class NeverDefers(LayerCommunicator): def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): @@ -236,8 +255,8 @@ class TestFfnExit(CustomTestCase): handoff leaves finish() untouched for the next layer's input norm.""" class Defers(LayerCommunicator): - def should_defer_moe_finalize(self, forward_batch, m=None): - return True + def _select_ffn_exit_fusions(self): + return (lambda forward_batch: comm.FfnExitFusion.DEFER_MOE_FINALIZE,) def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): return False diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index a059cd6528..49885106ee 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -38,7 +38,6 @@ from sglang.srt.layers.communicator import ( ) from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator from sglang.srt.layers.moe import utils as moe_utils -from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusionLayerCommunicator from sglang.srt.runtime_context import LoRABatchLayout from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -535,12 +534,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ) self.assertFalse(self.declared(build(direct, dp))) - def test_subclasses_that_pick_their_own_steps(self): - self.assertFalse(CuteDSLFusionLayerCommunicator._takes_declared_boundaries) - for cls in (LayerCommunicator, MHCLayerCommunicator): - with self.subTest(cls.__name__): - self.assertTrue(cls._takes_declared_boundaries) - def build_mhc( modes, parallel, *, a2a=False, dsa_cp=False, two_batch_overlap=False, **kwargs @@ -932,9 +925,13 @@ class TestTheLoraLayoutFollowsTheFfnInput(CustomTestCase): with ( planning(parallel, a2a=a2a), patch.object(comm, "get_exec", lambda: overlap), - # Otherwise the scatter-mode steps, which the subclasses that pick - # their own steps run. - patch.object(LayerCommunicator, "_takes_declared_boundaries", declared), + # Otherwise the scatter-mode steps, which a layer off the + # declarations runs. + nullcontext() + if declared + else patch.object( + LayerCommunicator, "_declared_sides", lambda self, **kwargs: None + ), ): communicator = LayerCommunicator( layer_scatter_modes=modes,