From bd37e7f513b67701afe27a93e1dc45320d724eeb Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:41:43 -0700 Subject: [PATCH] [Refactor] Choose a dense layer's boundaries under attention DP from both sides' declarations (#41257) --- python/sglang/srt/layers/boundary_layout.py | 91 ++- python/sglang/srt/layers/communicator.py | 245 ++++-- .../sglang/srt/layers/communicator_dsa_cp.py | 3 + python/sglang/srt/layers/communicator_mhc.py | 3 + .../srt/layers/moe/cutedsl_ar_fusion.py | 3 + .../unit/layers/test_communicator_ffn_exit.py | 15 +- .../unit/layers/test_communicator_layout.py | 2 + .../layers/test_declared_dense_boundary.py | 749 ++++++++++++++++++ .../test_layer_communicator_fusion_gate.py | 14 +- .../test_postprocess_reduce_scatterv.py | 26 +- .../test_aux_capture_deferred_allreduce.py | 6 +- 11 files changed, 1103 insertions(+), 54 deletions(-) create mode 100644 test/registered/unit/layers/test_declared_dense_boundary.py diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py index 9615c21b60..61bc0306ab 100644 --- a/python/sglang/srt/layers/boundary_layout.py +++ b/python/sglang/srt/layers/boundary_layout.py @@ -14,7 +14,7 @@ """Token layouts of the tensors handed across layer communication boundaries.""" from enum import Enum, auto -from typing import FrozenSet, Mapping +from typing import FrozenSet, Mapping, Optional import msgspec @@ -39,3 +39,92 @@ class Layout(msgspec.Struct, frozen=True): cls, *axes: TokenAxis, axis_sizes: Mapping[TokenAxis, int] ) -> "Layout": return cls(frozenset(axis for axis in axes if axis_sizes[axis] > 1)) + + +class SumGroup(Enum): + """The group a stage's output is summed over, by name: the handle comes + from get_parallel() when the sum runs.""" + + ATTN_TP = auto() + TP = auto() + # The group one all-reduce of a MoE output runs over. + MOE_OUTPUT = auto() + + +class StageInput(msgspec.Struct, frozen=True): + """The rows a stage's consumer needs: sharded over the token axes its + compute group does not span.""" + + layout: Layout + + +class StageOutput(msgspec.Struct, frozen=True): + """What a stage's producer hands the boundary after it, fixed at + construction: its rows, the group its output is summed over, and when it + leaves that sum to the boundary instead of completing it. + + What one output actually owes is carried with the value: a plain + tensor handed across layers is complete, an ``UnreducedOutput`` names what + is left. A raw compute output entering a boundary is read with this + declaration and the boundary's decision for the batch.""" + + layout: Layout + # None when there is nothing to sum. + group: Optional[SumGroup] = None + # The producer never reduces its output, e.g. a row-parallel projection + # built with reduce_results=False. + always_leaves: bool = False + # Otherwise it leaves the sum only when the boundary publishes the flag for + # it: fuse_mlp_allreduce, or mlp_reduce_scatter. + leaves_for_next_layer: bool = False + leaves_for_reduce_scatter: bool = False + # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that + # combine applies, as a MoE block does without any flag. + leaves_for_reduce_scatterv: bool = False + + +class DecoderLayerSides(msgspec.Struct, frozen=True): + """The declarations both sides of a decoder layer's boundaries are chosen + from.""" + + # The rows the layer takes and hands on; the residual stays on them. + layer_rows: Layout + attention: StageInput + attention_output: StageOutput + ffn: StageInput + ffn_output: StageOutput + + +def dense_decoder_layer_sides( + *, + axis_sizes: Mapping[TokenAxis, int], + leaves_for_next_layer: bool, + leaves_for_reduce_scatter: bool, +) -> DecoderLayerSides: + """An attention followed by a dense MLP on the TP group, derived from the + groups each computes over.""" + # Attention computes over the attention-TP ranks of one DP (and CP) shard. + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + # The TP group spans every token axis, so its MLP needs every row. + ffn = Layout.sharded_over(axis_sizes=axis_sizes) + attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 + return DecoderLayerSides( + layer_rows=attention, + attention=StageInput(attention), + # The output projection leaves the attention-TP sum to prepare_mlp. + attention_output=StageOutput( + attention, + group=SumGroup.ATTN_TP if attention_tp else None, + always_leaves=attention_tp, + ), + ffn=StageInput(ffn), + ffn_output=StageOutput( + ffn, + group=SumGroup.TP, + leaves_for_next_layer=leaves_for_next_layer, + leaves_for_reduce_scatter=leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=leaves_for_reduce_scatter, + ), + ) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index c65b4ae2cd..2dbda21fc6 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -37,7 +37,15 @@ from sglang.srt.layers.attention.dsa.utils import ( is_dsa_enable_prefill_cp, ) from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator -from sglang.srt.layers.boundary_layout import Layout, TokenAxis +from sglang.srt.layers.boundary_layout import ( + DecoderLayerSides, + Layout, + StageInput, + StageOutput, + SumGroup, + TokenAxis, + dense_decoder_layer_sides, +) from sglang.srt.layers.cp.utils import ( is_mla_cp_active, is_mla_cp_enabled, @@ -448,6 +456,9 @@ class LayerScatterModes: is_first_layer: bool = False # The model's last layer: its output goes to the final norm, not a next layer. is_last_layer: bool = False + # Whether the layer before this one has a sparse MLP; None when the modes + # were given directly, not planned from the layer sequence. + is_previous_layer_sparse: Optional[bool] = None @classmethod def init_new(cls, **kwargs): @@ -461,6 +472,7 @@ class LayerScatterModes: is_layer_sparse=context.is_layer_sparse, is_first_layer=context.layer_id == 0, is_last_layer=context.layer_id == context.num_layers - 1, + is_previous_layer_sparse=context.is_previous_layer_sparse, ) @classmethod @@ -708,6 +720,9 @@ def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: class LayerCommunicator: # Communicators built without __init__ (e.g. test doubles) publish no LoRA layout. _publish_lora_layout: bool = False + # Whether this class's boundary steps may be chosen from both sides' + # declarations; a subclass that picks its own steps says no. + _takes_declared_boundaries = True def __init__( self, @@ -738,15 +753,11 @@ class LayerCommunicator: self._context.force_layernorm_before_dp_gather = ( force_layernorm_before_dp_gather ) - self._post_init_communicate() - # Under attention DP, the base postprocess scatters the FFN output back - # to this rank's tokens, which the next layer's input can run instead; - # the MHC and DSA-CP postprocess do more, so theirs stays here. - self._postprocess_scatters_to_local_tokens = ( - self._communicate_summable_tensor_pair_fn - is CommunicateSummableTensorPairFn._scatter_hidden_states - ) - self._mlp_input, self._mlp_input_may_fuse = self._select_mlp_input() + sides = self._declared_sides() + if sides is not None: + self._select_declared_boundaries(sides) + else: + self._select_boundaries_from_scatter_modes() self._attn_input_fusions = self._select_attn_input_fusions() self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm @@ -760,6 +771,85 @@ class LayerCommunicator: # shard with no collectives while the region is active. self._sp_region = layernorm_sp.layernorm_sp_enabled() + def _declared_sides(self) -> Optional[DecoderLayerSides]: + """The declarations this layer's boundaries are chosen from: an + attention and a dense MLP on the TP group under attention DP, without CP + or LayerNorm SP, in a layer that takes the rows a dense layer or the + embedding hands on. None when the steps come from the scatter modes.""" + modes = self.layer_scatter_modes + parallel = get_parallel() + if not ( + self._takes_declared_boundaries + and parallel.attn_dp_size > 1 + and parallel.attn_cp_size == 1 + and not layernorm_sp.layernorm_sp_enabled() + and not modes.is_layer_sparse + and not enable_moe_dense_fully_dp() + and (modes.is_first_layer or modes.is_previous_layer_sparse is False) + ): + return None + return dense_decoder_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: parallel.attn_dp_size, + TokenAxis.ATTN_CP: parallel.attn_cp_size, + TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, + }, + leaves_for_next_layer=self.allow_deferred_ffn_reduction, + leaves_for_reduce_scatter=self.allow_reduce_scatter, + ) + + def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None: + """Choose the three boundary sides this layer owns from the + declarations: the attention input, the attention -> FFN steps, and the + FFN output's way back to the layer's rows.""" + self._communicate_simple_fn = _select_attention_input_move( + sides.layer_rows, sides.attention + ) + self._mlp_input = _select_ffn_input( + sides.attention_output, + residual=sides.layer_rows, + need=sides.ffn, + force_layernorm_before_gather=self.force_layernorm_before_dp_gather, + ) + # Neither fused kernel runs under attention DP. + self._mlp_input_may_fuse = False + self._ffn_output = sides.ffn_output + self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp( + sides.ffn_output, sides.layer_rows + ) + self._ffn_sum_is_movable = sides.ffn_output.group is not None + + def _select_boundaries_from_scatter_modes(self) -> None: + self._post_init_communicate() + # Under attention DP, the base postprocess scatters the FFN output back + # to this rank's tokens, which the next layer's input can run instead; + # the MHC and DSA-CP postprocess do more, so theirs stays here. + self._postprocess_scatters_to_local_tokens = ( + self._communicate_summable_tensor_pair_fn + is CommunicateSummableTensorPairFn._scatter_hidden_states + ) + modes = self.layer_scatter_modes + self._ffn_output = StageOutput( + self._context.layouts[modes.mlp_mode], + group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, + leaves_for_next_layer=self.allow_deferred_ffn_reduction, + leaves_for_reduce_scatter=self.allow_reduce_scatter, + # A MoE block leaves its sum to reduce_scatterv whenever it applies + # (should_skip_post_experts_all_reduce); a dense MLP does only under + # the published mlp_reduce_scatter. + leaves_for_reduce_scatterv=( + self.allow_reduce_scatter or modes.is_layer_sparse + ), + ) + # Whether the FFN sum is one the next layer's input can take: not when + # the way back is the MoE-CP scatter, nor when a SCATTERED FFN computes + # whole tokens with nothing left to sum. + self._ffn_sum_is_movable = modes.mlp_mode not in ( + ScatterMode.MOE_FULL, + ScatterMode.SCATTERED, + ) + self._mlp_input, self._mlp_input_may_fuse = self._select_mlp_input() + def _post_init_communicate(self): self._communicate_simple_fn = CommunicateSimpleFn.get_fn( input_mode=self.layer_scatter_modes.layer_input_mode, @@ -1092,6 +1182,9 @@ class LayerCommunicator: ): if self._in_sp_region(): return hidden_states, residual + if self._postprocess_scatters_to_local_tokens: + step = self._postprocess_dp_step(forward_batch) or _redistribute_output + return _to_local_tokens(step, forward_batch, hidden_states), residual return self._communicate_summable_tensor_pair_fn( hidden_states=hidden_states, residual=residual, @@ -1117,8 +1210,8 @@ class LayerCommunicator: return None return _reduce_and_redistribute_output_step( forward_batch, - allow_reduce_scatter=self.allow_reduce_scatter, - is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + leaves_for_reduce_scatter=self._ffn_output.leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=self._ffn_output.leaves_for_reduce_scatterv, ) def _ffn_leaves_sum_to_reduce_scatter( @@ -1127,7 +1220,7 @@ class LayerCommunicator: """Whether the FFN leaves its sum out because a reduce-scatter completes it: the attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" - if not self.allow_reduce_scatter: + if not self._ffn_output.leaves_for_reduce_scatter: return False if dp_step is not None: return True @@ -1141,9 +1234,7 @@ class LayerCommunicator: def ffn_reduction_group(self) -> GroupCoordinator: """The group this layer's FFN output owes its sum over: the MoE output's group on a sparse layer, the TP group a dense MLP reduces over.""" - if self.layer_scatter_modes.is_layer_sparse: - return post_experts_reduction_group() - return get_parallel().tp_group + return _sum_group(self._ffn_output.group) def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": """Decide once, before the FFN runs, what it skips and what completes its @@ -1157,7 +1248,7 @@ class LayerCommunicator: forward_batch=forward_batch, dp_step=dp_step, ) - if not self.allow_deferred_ffn_reduction: + if not self._ffn_output.leaves_for_next_layer: return FfnCompletion( defer_moe_finalize=False, fuse_mlp_allreduce=False, @@ -1255,14 +1346,9 @@ class LayerCommunicator: return False def _ffn_sum_can_move_to_next_layer(self) -> bool: - # When MOE_FULL is active (moe_cp allgather), fusion must be disabled because - # the fusion path skips postprocess_layer which contains the moe_cp scatter. - # Without scatter, hidden_states remain at MOE_FULL size while residual is at - # TP_ATTN_FULL size, causing a shape mismatch. - if ( - is_enable_moe_cp_allgather() - or self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL - ): + # Under the MoE-CP all-gather the fusion path would skip postprocess_layer + # and its MoE-CP scatter, leaving hidden_states longer than the residual. + if is_enable_moe_cp_allgather() or not self._ffn_sum_is_movable: return False # The fused residual+LN reduces over a single group. Hybrid EP+TP spans @@ -1284,12 +1370,7 @@ class LayerCommunicator: ): return False - if get_attn_tp_context().input_scattered: - return False - - # When mlp_mode is SCATTERED, the MLP runs on scattered data with no TP - # all-reduce, so there is nothing to fuse with the next layer. - return self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED + return not get_attn_tp_context().input_scattered # NOTE: This function will cause torch recompilation def should_fuse_mlp_allreduce_with_next_layer( @@ -1650,12 +1731,14 @@ def _mlp_input_dp_replicate( context: CommunicateContext, *, gathers_residual: bool, + reduces_attention_tp: bool, ): - """Attention DP: reduce, add and normalize locally, then gather.""" + """Attention DP: complete the attention-TP sum if it is owed, add and + normalize locally, then gather.""" if gathers_residual: residual = _redistribute_from_attn_tp_shards(residual) if hidden_states.shape[0] != 0: - if context.attn_tp_size > 1: + if reduces_attention_tp: hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) with use_symmetric_memory( get_parallel().tp_group, @@ -1702,10 +1785,79 @@ def _mlp_input_order( _mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions ) if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1: - return partial(_mlp_input_dp_replicate, gathers_residual=gathers_residual) + return partial( + _mlp_input_dp_replicate, + gathers_residual=gathers_residual, + reduces_attention_tp=context.attn_tp_size > 1, + ) return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual) +def _sum_group(group: SumGroup) -> GroupCoordinator: + parallel = get_parallel() + if group is SumGroup.ATTN_TP: + return parallel.attn_tp_group + if group is SumGroup.TP: + return parallel.tp_group + return post_experts_reduction_group() + + +def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: + """How the rows a layer takes become its attention's input.""" + if rows == need.layout: + return CommunicateSimpleFn._trivial + raise NotImplementedError(f"{rows=} {need=}") + + +def _select_ffn_input( + produced: StageOutput, + *, + residual: Layout, + need: StageInput, + force_layernorm_before_gather: bool, +) -> Callable: + """The steps from the attention output to the FFN input: complete the + attention-TP sum, add the residual and normalize, and gather the rows the + FFN's group needs, with the residual left on the attention's rows.""" + gathered = produced.layout.sharded - need.layout.sharded + if ( + residual != produced.layout + or not need.layout.sharded <= produced.layout.sharded + or gathered != {TokenAxis.ATTN_DP} + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + # What the attention output owes decides the steps: the attention-TP sum, + # always left by the output projection, or nothing. + owes_attention_tp = produced.group is SumGroup.ATTN_TP + if owes_attention_tp != produced.always_leaves or produced.group not in ( + None, + SumGroup.ATTN_TP, + ): + raise NotImplementedError(f"{produced=}") + # The partial order adds the residual on attention-TP rank 0 before the DP + # gather's collective completes that sum, which only a plain residual add + # allows. + if owes_attention_tp and not force_layernorm_before_gather: + return partial(_mlp_input_dp_partial, gathers_residual=False) + return partial( + _mlp_input_dp_replicate, + gathers_residual=False, + reduces_attention_tp=owes_attention_tp, + ) + + +def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool: + """Whether the FFN output goes back to the layer's rows by undoing the + attention-DP gather.""" + returned = rows.sharded - produced.layout.sharded + if not produced.layout.sharded <= rows.sharded or returned not in ( + frozenset(), + {TokenAxis.ATTN_DP}, + ): + raise NotImplementedError(f"{produced=} {rows=}") + return bool(returned) + + class MlpInputKind(Enum): """What the attention-TP -> FFN boundary does, given the two sides' layouts.""" @@ -1950,18 +2102,19 @@ def _redistribute_output_from_moe_cp( def _reduce_and_redistribute_output_step( - forward_batch: ForwardBatch, *, allow_reduce_scatter: bool, is_layer_sparse: bool + forward_batch: ForwardBatch, + *, + leaves_for_reduce_scatter: bool, + leaves_for_reduce_scatterv: bool, ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: - """The reduce-scatter that brings a FULL-layout layer output back to this - rank's tokens under attention DP when the FFN left its sum to it; None when - the FFN reduced the output and only a scatter remains.""" - # A MoE block leaves its sum to reduce_scatterv whenever it applies - # (should_skip_post_experts_all_reduce); a dense MLP does only under the - # published mlp_reduce_scatter, which needs allow_reduce_scatter. - if should_use_dp_reduce_scatterv() and (allow_reduce_scatter or is_layer_sparse): + """The reduce-scatter that brings an FFN output gathered over attention + DP back to this rank's tokens when the FFN leaves its sum to it (see + StageOutput); None when the FFN reduces the output and only a scatter + remains.""" + if should_use_dp_reduce_scatterv() and leaves_for_reduce_scatterv: return _reduce_and_redistribute_output_varlen if ( - allow_reduce_scatter + leaves_for_reduce_scatter and forward_batch.dp_padding_mode.is_max_len() and can_use_dp_reduce_scatter() ): @@ -2053,8 +2206,10 @@ class CommunicateSummableTensorPairFn: step = ( _reduce_and_redistribute_output_step( forward_batch, - allow_reduce_scatter=allow_reduce_scatter, - is_layer_sparse=is_layer_sparse, + leaves_for_reduce_scatter=allow_reduce_scatter, + # A MoE block leaves its sum to reduce_scatterv whenever it + # applies (should_skip_post_experts_all_reduce). + leaves_for_reduce_scatterv=allow_reduce_scatter or is_layer_sparse, ) or _redistribute_output ) diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index ac762fa0ed..aad1f67778 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -84,6 +84,9 @@ def dsa_cp_reduce_scatter_hidden_states(hidden_states: torch.Tensor): class DSACPLayerCommunicator(LayerCommunicator): + # Chooses its own boundary steps, not from the declarations. + _takes_declared_boundaries = False + def _post_init_communicate(self): # SCATTERED in attn tp is different from SCATTERED in global tp when dp_size > 1 if self.layer_scatter_modes.mlp_mode != ScatterMode.SCATTERED: diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index 64e9c94634..ec37e36372 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -384,6 +384,9 @@ class MHCCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn): class MHCLayerCommunicator(LayerCommunicator): + # Chooses its own boundary steps, not from the declarations. + _takes_declared_boundaries = False + def __init__( self, layer_scatter_modes: LayerScatterModes, diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index 3e53da15b6..1cf9b5f57a 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -176,6 +176,9 @@ class CuteDSLFusionService: class CuteDSLFusionLayerCommunicator(LayerCommunicator): + # Chooses its own boundary steps, not from the declarations. + _takes_declared_boundaries = False + fusion_service: CuteDSLFusionService | None = None # The runner can defer and a successor or the final norm consumes the handoff. diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 47d7371c6b..92f114d620 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -8,6 +8,7 @@ import torch import sglang from sglang.srt.layers import communicator as comm +from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup from sglang.srt.layers.communicator import ( LayerCommunicator, UnreducedOutput, @@ -38,7 +39,9 @@ def make_communicator( """A communicator whose decisions and postprocess are stubbed, built without the process-wide parallel state.""" communicator = cls.__new__(cls) - communicator.allow_deferred_ffn_reduction = allow_deferred + communicator._ffn_output = StageOutput( + Layout(frozenset()), group=SumGroup.TP, leaves_for_next_layer=allow_deferred + ) if cls is LayerCommunicator: communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse) else: @@ -280,12 +283,16 @@ class TestSelectFfnCompletion(CustomTestCase): self, *, fuse=False, is_last_layer=False, scatters=True, sp_region=False ): communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator.allow_deferred_ffn_reduction = True communicator.is_last_layer = is_last_layer communicator._sp_region = sp_region communicator._postprocess_scatters_to_local_tokens = scatters - communicator.allow_reduce_scatter = True - communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) + communicator._ffn_output = StageOutput( + Layout(frozenset()), + group=SumGroup.MOE_OUTPUT, + leaves_for_next_layer=True, + leaves_for_reduce_scatter=True, + leaves_for_reduce_scatterv=True, + ) communicator._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: fuse communicator._ffn_leaves_sum_to_reduce_scatter = lambda forward_batch, dp_step: ( not fuse diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py index 06989a939e..781a42518a 100644 --- a/test/registered/unit/layers/test_communicator_layout.py +++ b/test/registered/unit/layers/test_communicator_layout.py @@ -202,6 +202,7 @@ class TestMlpInputOrder(CustomTestCase): comm._mlp_input_order(make_context(dp=2), TP_ATTN_FULL, fusions), comm._mlp_input_dp_replicate, gathers_residual=False, + reduces_attention_tp=False, ) dp_tp = make_context(dp=2, tp=2) self.assert_order( @@ -214,6 +215,7 @@ class TestMlpInputOrder(CustomTestCase): comm._mlp_input_order(dp_tp, SCATTERED, fusions), comm._mlp_input_dp_replicate, gathers_residual=True, + reduces_attention_tp=True, ) def test_a_gather_runs_the_order_chosen_at_construction(self): diff --git a/test/registered/unit/layers/test_declared_dense_boundary.py b/test/registered/unit/layers/test_declared_dense_boundary.py new file mode 100644 index 0000000000..9ba2111169 --- /dev/null +++ b/test/registered/unit/layers/test_declared_dense_boundary.py @@ -0,0 +1,749 @@ +"""The boundaries of an attention and a dense MLP on the TP group under attention +DP, chosen from both sides' declarations. + +The numeric checks run every rank of a DP x attention-TP world as a thread over +fake collectives that wait for all members of their group, build the layers' +communicators for real, and pass them scatter modes that fail on any read of a +mode field. +""" + +import itertools +import threading +import unittest +from contextlib import ExitStack, contextmanager, nullcontext +from functools import partial +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers import communicator as comm +from sglang.srt.layers.boundary_layout import ( + SumGroup, + TokenAxis, + dense_decoder_layer_sides, +) +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, + ScatterMode, + UnreducedOutput, + scatter_mode_layouts, +) +from sglang.srt.layers.communicator_dsa_cp import DSACPLayerCommunicator +from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator +from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusionLayerCommunicator +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +HIDDEN = 4 + + +def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): + fields = dict( + tp_size=attn_dp * attn_cp * attn_tp, + tp_rank=0, + attn_dp_size=attn_dp, + attn_tp_size=attn_tp, + attn_tp_rank=0, + attn_cp_size=attn_cp, + attn_cp_rank=0, + moe_dense_tp_size=None, + enable_prefill_cp=False, + moe_ep_size=1, + moe_tp_size=attn_dp * attn_cp * attn_tp, + moe_dp_size=1, + dwdp_size=1, + enable_dp_attention=attn_dp > 1, + tp_group=SimpleNamespace(name="tp"), + attn_tp_group=SimpleNamespace(name="attn_tp"), + ) + fields.update(overrides) + return SimpleNamespace(**fields) + + +@contextmanager +def planning(parallel, *, sp=False): + """What layer planning and communicator construction read, without the + process-wide parallel state. ``parallel`` may be a callable, for a + per-thread parallel state.""" + get_parallel = parallel if callable(parallel) else (lambda: parallel) + with ( + patch.object(comm, "get_parallel", get_parallel), + patch.object(comm, "is_dsa_enable_prefill_cp", lambda: False), + patch.object(comm, "is_mla_cp_enabled", lambda: False), + patch.object(comm, "get_moe_cp_size", lambda: 1), + patch.object(comm.layernorm_sp, "layernorm_sp_enabled", lambda: sp), + patch.object( + comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) + ), + patch.object( + comm, "get_moe_a2a_backend", lambda: SimpleNamespace(is_none=lambda: True) + ), + patch.object( + comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False + ), + patch.object(comm, "is_enable_moe_cp_allgather", lambda: False), + patch.object(comm, "get_lora", lambda: SimpleNamespace(enable_lora=False)), + ): + yield + + +class Norm: + """add + norm stand-in: (2 * (h + r), h + r); one input: 2 * h.""" + + def __call__(self, x, residual=None, post_residual_addition=None): + assert post_residual_addition is None + if residual is None: + return x * 2 + s = x + residual + return s * 2, s + + +class FactsOnly: + """Scatter modes whose mode fields fail when read: only the layer facts + are there.""" + + def __init__(self, **facts): + self.__dict__.update(facts) + + def __getattr__(self, name): + raise AssertionError(f"read scatter mode {name!r}") + + +def dense_layer_facts(layer_id, num_layers, *, previous_sparse=False): + return FactsOnly( + is_layer_sparse=False, + is_first_layer=layer_id == 0, + is_last_layer=layer_id == num_layers - 1, + is_previous_layer_sparse=None if layer_id == 0 else previous_sparse, + ) + + +def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs): + with planning(parallel, sp=sp): + return cls( + layer_scatter_modes=modes, + input_layernorm=Norm(), + post_attention_layernorm=Norm(), + **kwargs, + ) + + +def planned_modes(layer_id, num_layers, *, sparse, previous_sparse, parallel): + with planning(parallel): + return LayerScatterModes.init_new( + layer_id=layer_id, + num_layers=num_layers, + is_layer_sparse=sparse, + is_previous_layer_sparse=previous_sparse, + is_next_layer_sparse=False, + ) + + +class TestDeclarationsMatchScatterModes(CustomTestCase): + """The layouts derived from the groups each side computes over are the + layouts of the modes layer planning gives the same layers.""" + + def test_every_attention_dp_and_tp(self): + for attn_dp, attn_tp, layer_id in itertools.product( + (2, 4, 8), (1, 2, 4), (0, 1, 3) + ): + with self.subTest(attn_dp=attn_dp, attn_tp=attn_tp, layer_id=layer_id): + parallel = parallel_of(attn_dp=attn_dp, attn_tp=attn_tp) + modes = planned_modes( + layer_id, 4, sparse=False, previous_sparse=False, parallel=parallel + ) + sizes = { + TokenAxis.ATTN_DP: attn_dp, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: attn_tp, + } + layouts = scatter_mode_layouts( + attn_dp_size=attn_dp, attn_cp_size=1, attn_tp_size=attn_tp + ) + sides = dense_decoder_layer_sides( + axis_sizes=sizes, + leaves_for_next_layer=True, + leaves_for_reduce_scatter=True, + ) + self.assertEqual(sides.layer_rows, layouts[modes.layer_input_mode]) + self.assertEqual(sides.layer_rows, layouts[modes.middle_residual_mode]) + self.assertEqual(sides.layer_rows, layouts[modes.layer_output_mode]) + self.assertEqual(sides.attention.layout, layouts[modes.attn_mode]) + self.assertEqual(sides.ffn.layout, layouts[modes.mlp_mode]) + self.assertEqual(sides.attention_output.layout, sides.layer_rows) + self.assertEqual(sides.ffn_output.layout, sides.ffn.layout) + + def test_the_attention_output_owes_the_attention_tp_sum(self): + for attn_tp in (1, 2): + sides = dense_decoder_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 2, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: attn_tp, + }, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + ) + owed = attn_tp > 1 + self.assertIs( + sides.attention_output.group, SumGroup.ATTN_TP if owed else None + ) + self.assertIs(sides.attention_output.always_leaves, owed) + self.assertIs(sides.ffn_output.group, SumGroup.TP) + self.assertFalse(sides.ffn_output.always_leaves) + + +class TestWhichLayersUseDeclarations(CustomTestCase): + """The layer is the unit: all three boundary sides it owns follow the + declarations, or none do.""" + + def declared(self, communicator): + return communicator._mlp_input.func in ( + comm._mlp_input_dp_partial, + comm._mlp_input_dp_replicate, + ) + + def test_a_dense_model(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + for layer_id in range(3): + with self.subTest(layer_id=layer_id): + communicator = build(dense_layer_facts(layer_id, 3), parallel) + self.assertTrue(self.declared(communicator)) + self.assertIs( + communicator._communicate_simple_fn, + comm.CommunicateSimpleFn._trivial, + ) + self.assertTrue(communicator._postprocess_scatters_to_local_tokens) + + def test_the_order_follows_the_attention_output(self): + for attn_tp, force, order in ( + (2, False, comm._mlp_input_dp_partial), + (2, True, comm._mlp_input_dp_replicate), + (1, False, comm._mlp_input_dp_replicate), + ): + with self.subTest(attn_tp=attn_tp, force=force): + communicator = build( + dense_layer_facts(1, 3), + parallel_of(attn_dp=2, attn_tp=attn_tp), + force_layernorm_before_dp_gather=force, + ) + self.assertIs(communicator._mlp_input.func, order) + + def test_a_dense_first_layer_before_sparse_ones(self): + # DeepSeek-V2-Lite: layer 0 is dense, the rest sparse. + parallel = parallel_of(attn_dp=2, attn_tp=2) + first = build( + planned_modes(0, 3, sparse=False, previous_sparse=False, parallel=parallel), + parallel, + allow_reduce_scatter=True, + ) + second = build( + planned_modes(1, 3, sparse=True, previous_sparse=False, parallel=parallel), + parallel, + allow_reduce_scatter=True, + ) + self.assertTrue(self.declared(first)) + self.assertFalse(self.declared(second)) + self.assertIs(second._mlp_input.func, comm._mlp_input_gather) + + def test_layers_that_keep_the_scatter_modes(self): + dp = parallel_of(attn_dp=2, attn_tp=2) + cases = { + "no attention DP": ( + dense_layer_facts(1, 3), + parallel_of(attn_dp=1, attn_tp=4), + ), + "attention CP": ( + dense_layer_facts(1, 3), + parallel_of(attn_dp=2, attn_tp=1, attn_cp=2), + ), + "dense MLP fully DP": ( + dense_layer_facts(1, 3), + parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1), + ), + "after a sparse layer": (dense_layer_facts(1, 3, previous_sparse=True), dp), + } + for name, (facts, parallel) in cases.items(): + with self.subTest(name): + with planning(parallel): + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator.layer_scatter_modes = facts + communicator.allow_deferred_ffn_reduction = True + communicator.allow_reduce_scatter = False + self.assertIsNone(communicator._declared_sides()) + # Modes given directly (Nemotron-H's stages, the SP sibling) do not say + # which rows the layer takes. + direct = LayerScatterModes( + layer_input_mode=ScatterMode.TP_ATTN_FULL, + attn_mode=ScatterMode.TP_ATTN_FULL, + mlp_mode=ScatterMode.FULL, + middle_residual_mode=ScatterMode.TP_ATTN_FULL, + layer_output_mode=ScatterMode.TP_ATTN_FULL, + ) + self.assertFalse(self.declared(build(direct, dp))) + with planning(dp, sp=True): + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator.layer_scatter_modes = dense_layer_facts(1, 3) + self.assertIsNone(communicator._declared_sides()) + + def test_subclasses_that_pick_their_own_steps(self): + for cls in ( + MHCLayerCommunicator, + DSACPLayerCommunicator, + CuteDSLFusionLayerCommunicator, + ): + with self.subTest(cls.__name__): + self.assertFalse(cls._takes_declared_boundaries) + self.assertTrue(LayerCommunicator._takes_declared_boundaries) + + +class TestTheAttentionOutputDecidesItsSum(CustomTestCase): + """prepare_mlp completes the attention-TP sum exactly when the attention + output's declaration says it is owed, whatever the attention-TP size.""" + + def run_steps(self, produced, *, force=False): + sizes = { + TokenAxis.ATTN_DP: 2, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + } + sides = dense_decoder_layer_sides( + axis_sizes=sizes, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + ) + steps = comm._select_ffn_input( + produced, + residual=sides.layer_rows, + need=sides.ffn, + force_layernorm_before_gather=force, + ) + reduced = [] + context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0) + with ( + patch.object( + comm, + "attention_tensor_model_parallel_all_reduce", + lambda x: reduced.append(x) or 2 * x, + ), + patch.object(comm, "_redistribute_input_to_dp", lambda h, fb: h), + patch.object( + comm, "get_parallel", lambda: parallel_of(attn_dp=2, attn_tp=2) + ), + patch.object( + comm, "use_symmetric_memory", lambda g, disabled=False: nullcontext() + ), + patch.object(comm, "is_allocation_symmetric", lambda: False), + ): + hidden, residual = steps( + torch.full((1, HIDDEN), 7.0), + torch.full((1, HIDDEN), 3.0), + None, + Norm(), + context, + ) + return len(reduced), hidden, residual + + def test_a_complete_output_is_not_summed_again(self): + layout = dense_decoder_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 2, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + }, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + ).layer_rows + complete = comm.StageOutput(layout) + for force in (False, True): + with self.subTest(force=force): + reductions, hidden, residual = self.run_steps(complete, force=force) + self.assertEqual(reductions, 0) + torch.testing.assert_close(residual, torch.full((1, HIDDEN), 10.0)) + torch.testing.assert_close(hidden, torch.full((1, HIDDEN), 20.0)) + + def test_an_owed_output_is_summed_once(self): + layout = dense_decoder_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 2, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + }, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + ).layer_rows + owed = comm.StageOutput(layout, group=SumGroup.ATTN_TP, always_leaves=True) + reductions, hidden, residual = self.run_steps(owed, force=True) + self.assertEqual(reductions, 1) + # The stand-in all-reduce doubles the one rank's value. + torch.testing.assert_close(residual, torch.full((1, HIDDEN), 17.0)) + + def test_declarations_the_steps_cannot_run_are_rejected(self): + layout = dense_decoder_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 2, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + }, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + ).layer_rows + for produced in ( + # Leaves the attention-TP sum only when asked, like a mixer exit. + comm.StageOutput(layout, group=SumGroup.ATTN_TP), + # Always leaves a sum over no group. + comm.StageOutput(layout, always_leaves=True), + # Owes a sum over a group the steps do not complete. + comm.StageOutput(layout, group=SumGroup.TP, always_leaves=True), + ): + with ( + self.subTest(produced=produced), + self.assertRaises(NotImplementedError), + ): + self.run_steps(produced) + + +# --------------------------------------------------------------------------- +# Two consecutive dense layers, rank by rank. + + +class World: + def __init__(self, size): + self.size = size + self.local = threading.local() + self.groups = [] + + def state(self): + return self.local.state + + def run(self, states, fn): + results, errors = [None] * self.size, [None] * self.size + + def body(rank): + self.local.state = states[rank] + try: + results[rank] = fn(rank) + except BaseException as error: # noqa: BLE001 + errors[rank] = error + for group in self.groups: + group.barrier.abort() + + threads = [threading.Thread(target=body, args=(r,)) for r in range(self.size)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return results, errors + + +class Group: + """A process group whose collectives wait for every member thread.""" + + def __init__(self, world, name, ranks): + self.world, self.name, self.ranks = world, name, list(ranks) + self.barrier = threading.Barrier(len(self.ranks), timeout=30) + self.slots = {} + world.groups.append(self) + + @property + def world_size(self): + return len(self.ranks) + + @property + def rank_in_group(self): + return self.ranks.index(self.world.state().rank) + + def _exchange(self, x): + self.world.state().calls.append(self.name) + self.slots[self.world.state().rank] = x.clone() + self.barrier.wait() + values = [self.slots[r] for r in self.ranks] + self.barrier.wait() + return values + + def all_reduce(self, x): + return torch.stack(self._exchange(x)).sum(0) + + def reduce_scatter_tensor(self, output, input): + full = torch.stack(self._exchange(input)).sum(0) + output.copy_(full.tensor_split(self.world_size)[self.rank_in_group]) + + def all_gather_into_tensor(self, output, input): + output.copy_(torch.cat(self._exchange(input))) + + def reduce_scatterv(self, input, output=None, sizes=None): + full = torch.stack(self._exchange(input)).sum(0) + output.copy_(full.split(list(sizes))[self.rank_in_group]) + return output + + +class Flags: + """The per-forward flags an FFN exit publishes, one set per rank.""" + + def __init__(self): + self.fuse_mlp_allreduce = False + self.mlp_reduce_scatter = False + self.defer_moe_finalize = False + self.sp_active = False + + @contextmanager + def scoped(self, **flags): + saved = {k: getattr(self, k) for k in flags} + self.__dict__.update(flags) + try: + yield + finally: + self.__dict__.update(saved) + + +def fake_dp_gather(global_tokens, local_tokens, forward_batch, *, is_partial): + state = WORLD[0].state() + global_tokens.zero_() + if is_partial or state.parallel.attn_tp_rank == 0: + global_tokens[state.offset : state.offset + state.rows] += local_tokens[ + : state.rows + ] + global_tokens.copy_(state.parallel.tp_group.all_reduce(global_tokens)) + + +def fake_dp_scatter(local_tokens, global_tokens, forward_batch): + state = WORLD[0].state() + local_tokens.fill_(0) + local_tokens[: state.rows] = global_tokens[state.offset : state.offset + state.rows] + + +def fake_dp_reduce_scatter_tensor(output, input): + # dp_attention.dp_reduce_scatter_tensor for max-len padding. + parallel = WORLD[0].state().parallel + if parallel.tp_size == parallel.attn_dp_size: + parallel.tp_group.reduce_scatter_tensor(output, input) + else: + chunk = input.tensor_split(parallel.tp_size)[parallel.tp_rank].clone() + parallel.tp_group.reduce_scatter_tensor(chunk, input) + parallel.attn_tp_group.all_gather_into_tensor(output, chunk) + + +WORLD = [None] + + +def state(): + return WORLD[0].state() + + +@contextmanager +def running(*, reduce_scatterv): + replaced = { + "get_forward": lambda: state().flags, + "attention_tensor_model_parallel_all_reduce": lambda x: ( + state().parallel.attn_tp_group.all_reduce(x) + ), + "get_global_dp_buffer": lambda g: torch.zeros( + state().global_rows, HIDDEN, dtype=torch.double + ), + "get_local_dp_buffer": lambda g: torch.zeros( + state().local_rows, HIDDEN, dtype=torch.double + ), + "dp_gather_replicate": partial(fake_dp_gather, is_partial=False), + "dp_gather_partial": partial(fake_dp_gather, is_partial=True), + "dp_scatter": fake_dp_scatter, + "dp_reduce_scatter_tensor": fake_dp_reduce_scatter_tensor, + "get_dp_global_num_tokens": lambda: state().dp_rows, + "should_use_dp_reduce_scatterv": lambda: reduce_scatterv, + "can_use_dp_reduce_scatter": lambda: True, + "is_dp_attention_enabled": lambda: True, + "apply_flashinfer_allreduce_fusion": lambda n: False, + "post_experts_output_is_complete": lambda **kw: False, + "post_experts_reduction_group": lambda: state().parallel.tp_group, + "get_lora": lambda: SimpleNamespace(enable_lora=False), + "get_exec": lambda: SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=False) + ), + "get_attn_tp_context": lambda: SimpleNamespace( + input_scattered=False, set_attn_inputs=lambda x: None + ), + "use_symmetric_memory": lambda group, disabled=False: nullcontext(), + "is_allocation_symmetric": lambda: False, + } + with ExitStack() as stack: + stack.enter_context(planning(lambda: state().parallel)) + for name, value in replaced.items(): + stack.enter_context(patch.object(comm, name, value)) + yield + + +# Binary fractions, so partial sums add back to the value exactly. +WEIGHTS = {1: [1.0], 2: [0.25, 0.75], 4: [0.125, 0.25, 0.375, 0.25]} + + +def attention(x, state): + """The attention output: 3 * x, as this rank's attention-TP partial.""" + return 3 * x * WEIGHTS[state.parallel.attn_tp_size][state.parallel.attn_tp_rank] + + +def dense_mlp(x, state): + """A dense MLP on the TP group: 5 * x, reduced unless a published flag asks + it to leave the sum, as RowParallelLinear does.""" + partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank] + if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter: + return partial_sum + return state.parallel.tp_group.all_reduce(partial_sum) + + +def reference(x): + """(hidden, residual) after two layers of Norm / attention / dense MLP.""" + residual = x + for _ in range(2): + hidden = 2 * residual + residual = 3 * hidden + residual + ffn = 5 * (2 * residual) + residual = ffn + residual + # The last layer hands on its FFN output and the residual before it. + return ffn, residual - ffn + + +class TestTwoDenseLayers(CustomTestCase): + def run_world(self, *, attn_dp, attn_tp, rows, padding, reduce_scatterv, leaves): + tp = attn_dp * attn_tp + world = World(tp) + WORLD[0] = world + tp_group = Group(world, "tp", range(tp)) + attn_tp_groups = [ + tp_group + if attn_tp == tp + else Group(world, f"attn_tp{d}", range(d * attn_tp, (d + 1) * attn_tp)) + for d in range(attn_dp) + ] + max_len = padding == "max_len" + # Max-len padding pads every DP rank to one length that attention TP tiles. + local_len = -(-max(rows) // attn_tp) * attn_tp if max_len else None + offsets = ( + [d * local_len for d in range(attn_dp)] + if max_len + else [sum(rows[:d]) for d in range(attn_dp)] + ) + global_rows = attn_dp * local_len if max_len else sum(rows) + generator = torch.Generator().manual_seed(0) + embeddings = [ + torch.randint(-4, 4, (n, HIDDEN), generator=generator).double() + for n in rows + ] + forward_batch = SimpleNamespace( + forward_mode=SimpleNamespace( + is_context_parallel_extend=lambda: False, + is_decode_or_idle=lambda: True, + ), + dp_padding_mode=SimpleNamespace(is_max_len=lambda: max_len), + global_dp_buffer_len=global_rows, + ) + states = [] + for rank in range(tp): + d, t = divmod(rank, attn_tp) + states.append( + SimpleNamespace( + rank=rank, + parallel=parallel_of( + attn_dp=attn_dp, + attn_tp=attn_tp, + tp_rank=rank, + attn_tp_rank=t, + tp_group=tp_group, + attn_tp_group=attn_tp_groups[d], + ), + flags=Flags(), + calls=[], + dp=d, + rows=rows[d], + offset=offsets[d], + local_rows=local_len or rows[d], + global_rows=global_rows, + dp_rows=[local_len] * attn_dp if max_len else list(rows), + ) + ) + + def forward(rank): + s = states[rank] + layers = [ + LayerCommunicator( + layer_scatter_modes=dense_layer_facts(i, 2), + input_layernorm=Norm(), + post_attention_layernorm=Norm(), + allow_reduce_scatter=leaves, + allow_deferred_ffn_reduction=leaves, + ) + for i in range(2) + ] + hidden = torch.zeros(s.local_rows, HIDDEN).double() + hidden[: s.rows] = embeddings[s.dp] + residual = None + handed_on = [] + for layer in layers: + hidden, residual = layer.prepare_attn(hidden, residual, forward_batch) + hidden = attention(hidden, s) + hidden, residual = layer.prepare_mlp(hidden, residual, forward_batch) + with layer.ffn_exit(forward_batch) as ffn_exit: + hidden = dense_mlp(hidden, s) + hidden, residual = ffn_exit.finish(hidden, residual) + handed_on.append(type(hidden)) + hidden, residual = layers[-1].finish_layer_stack( + hidden, residual, forward_batch + ) + return hidden[: s.rows], residual[: s.rows], handed_on + + with running(reduce_scatterv=reduce_scatterv): + results, errors = world.run(states, forward) + for rank, error in enumerate(errors): + if error is not None: + raise AssertionError(f"rank {rank} raised") from error + for rank, (hidden, residual, handed_on) in enumerate(results): + want_hidden, want_residual = reference(embeddings[states[rank].dp]) + torch.testing.assert_close(hidden, want_hidden, rtol=0, atol=0) + torch.testing.assert_close(residual, want_residual, rtol=0, atol=0) + # Every rank ran the same collectives. + self.assertEqual(len({tuple(s.calls) for s in states if s.dp == 0}), 1) + return results + + def test_rows_come_back_to_each_rank(self): + for (attn_dp, attn_tp), rows, padding, leaves in itertools.product( + ((2, 2), (4, 1), (2, 1)), + ("ragged", "idle"), + ("sum_len", "max_len"), + (False, True), + ): + token_rows = { + ("ragged", 2): [3, 1], + ("idle", 2): [2, 0], + ("ragged", 4): [2, 3, 1, 1], + ("idle", 4): [2, 0, 3, 1], + }[rows, attn_dp] + for reduce_scatterv in (False, True) if attn_tp == 1 else (False,): + with self.subTest( + attn_dp=attn_dp, + attn_tp=attn_tp, + rows=token_rows, + padding=padding, + leaves=leaves, + reduce_scatterv=reduce_scatterv, + ): + results = self.run_world( + attn_dp=attn_dp, + attn_tp=attn_tp, + rows=token_rows, + padding=padding, + reduce_scatterv=reduce_scatterv, + leaves=leaves, + ) + first_layer_hands_on = results[0][2][0] + # A producer that may leave its sum leaves it for the next + # layer's input; one that may not hands on a complete value. + self.assertIs( + first_layer_hands_on, + UnreducedOutput if leaves else torch.Tensor, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 461be20005..6020292ce4 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers import communicator as comm +from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, @@ -24,6 +25,11 @@ def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._speculative_algo = None communicator.layer_scatter_modes = types.SimpleNamespace(mlp_mode=mlp_mode) + # Fixed at construction by the scatter-mode path from the MLP's mode. + communicator._ffn_sum_is_movable = mlp_mode not in ( + ScatterMode.MOE_FULL, + ScatterMode.SCATTERED, + ) communicator.is_last_layer = False communicator._context = types.SimpleNamespace(tp_size=4) return communicator @@ -324,8 +330,12 @@ class TestDeferFfnReduction(CustomTestCase): communicator.is_last_layer = is_last_layer communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._sp_region = sp_active - communicator.allow_reduce_scatter = True - communicator.layer_scatter_modes.is_layer_sparse = True + communicator._ffn_output = StageOutput( + Layout(frozenset()), + group=SumGroup.MOE_OUTPUT, + leaves_for_reduce_scatter=True, + leaves_for_reduce_scatterv=True, + ) forward_batch = types.SimpleNamespace( input_ids=types.SimpleNamespace(shape=(batch_size,)), global_dp_buffer_len=global_tokens, diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index eb398b025b..2769a055e3 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -142,6 +142,7 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._sp_region = False communicator._context = None + communicator._postprocess_scatters_to_local_tokens = False communicator.allow_reduce_scatter = False communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) communicator._communicate_summable_tensor_pair_fn = lambda **kwargs: ( @@ -150,6 +151,26 @@ class TestPostprocessReduceScatterv(CustomTestCase): communicator.postprocess_layer(None, None, None) self.assertIs(seen["is_layer_sparse"], True) + def test_postprocess_takes_the_output_back_with_the_exit_step(self): + # Under attention DP the base postprocess brings the FFN output back with + # the step the FFN exit chooses. + for step in (object(), None): + with self.subTest(reduce_scatter=step is not None): + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator._sp_region = False + communicator._postprocess_scatters_to_local_tokens = True + communicator._postprocess_dp_step = lambda forward_batch: step + with patch.object( + comm, "_to_local_tokens", side_effect=lambda s, fb, h: (s, h) + ): + hidden_states, residual = communicator.postprocess_layer( + "h", "r", "fb" + ) + self.assertEqual( + hidden_states, (step or comm._redistribute_output, "h") + ) + self.assertEqual(residual, "r") + class TestReduceAndRedistributeOutputStep(CustomTestCase): """The reduce-scatter that brings a FULL-layout output back to this rank's @@ -163,8 +184,11 @@ class TestReduceAndRedistributeOutputStep(CustomTestCase): patch.object(comm, "should_use_dp_reduce_scatterv", return_value=varlen), patch.object(comm, "can_use_dp_reduce_scatter", return_value=tiles), ): + # A MoE block leaves its sum to reduce_scatterv whenever it applies. return comm._reduce_and_redistribute_output_step( - forward_batch, allow_reduce_scatter=allow, is_layer_sparse=sparse + forward_batch, + leaves_for_reduce_scatter=allow, + leaves_for_reduce_scatterv=allow or sparse, ) def test_every_condition(self): diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 7b2736d05f..f52516e334 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -46,7 +46,11 @@ class DeferringLayer(nn.Module): super().__init__() self.return_topk = return_topk self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - self.layer_communicator.allow_deferred_ffn_reduction = True + self.layer_communicator._ffn_output = comm.StageOutput( + comm.Layout(frozenset()), + group=comm.SumGroup.TP, + leaves_for_next_layer=True, + ) self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = ( lambda batch: defer )