From 3450d68d4ede9c5c8fbd7f56aba6b28e57630d9b Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:50:35 -0700 Subject: [PATCH] [Refactor] Decide an FFN exit's completion once and declare the group it owes (#41200) --- python/sglang/srt/layers/communicator.py | 243 ++++++++++-------- .../srt/layers/flashinfer_comm_fusion.py | 9 +- python/sglang/srt/layers/moe/__init__.py | 2 + python/sglang/srt/layers/moe/utils.py | 27 +- python/sglang/srt/models/nemotron_h.py | 8 +- .../unit/layers/test_communicator_ffn_exit.py | 148 ++++++----- .../test_layer_communicator_fusion_gate.py | 52 ++-- .../layers/test_layernorm_allreduce_fusion.py | 17 ++ .../test_postprocess_reduce_scatterv.py | 9 +- .../unit/layers/test_prepare_attn_steps.py | 44 +++- .../test_aux_capture_deferred_allreduce.py | 13 +- .../test_deepstack_deferred_allreduce.py | 16 +- 12 files changed, 346 insertions(+), 242 deletions(-) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 89a772bb7a..c19a783bbf 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -66,9 +66,9 @@ from sglang.srt.layers.flashinfer_comm_fusion import ( ) from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, - deferred_post_experts_all_reduce, get_moe_a2a_backend, post_experts_output_is_complete, + post_experts_reduction_group, should_use_dp_reduce_scatterv, should_use_flashinfer_cutlass_moe_fp4_allgather, ) @@ -629,13 +629,15 @@ def _attn_input_update_and_read_residual(quant_format: str): class UnreducedOutput(msgspec.Struct, frozen=True): - """A layer output that still owes its FFN all-reduce, left for the next - layer's input. Hand it to the next layer, or pass it through reduce_output() - before reading it any other way.""" + """A layer output that still owes its sum, left for the next layer's input. + Hand it to the next layer, or pass it through reduce_output() before reading + it any other way. The producer says what is owed: one all-reduce over + ``group`` that keeps the layout, or ``reduce_and_redistribute``.""" partial: torch.Tensor - # Under attention DP: the reduce-scatter that sums ``partial`` over TP while - # bringing it back to this rank's tokens, in place of the all-reduce. + group: Optional[GroupCoordinator] = None + # Under attention DP: the reduction that also brings ``partial`` back to this + # rank's tokens (a reduce-scatter, or an all-reduce then a scatter). reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None @@ -646,7 +648,7 @@ def reduce_output( if isinstance(hidden_states, UnreducedOutput): if hidden_states.reduce_and_redistribute is not None: return hidden_states.reduce_and_redistribute(hidden_states.partial) - return deferred_post_experts_all_reduce(hidden_states.partial) + return hidden_states.group.all_reduce(hidden_states.partial) return hidden_states @@ -666,14 +668,13 @@ def _batch_size(forward_batch: ForwardBatch) -> int: def _deferred_reduction_runs_on_the_tp_group() -> bool: - """Whether deferred_post_experts_all_reduce reduces over the TP group object + """Whether an MoE output's one all-reduce runs over the TP group object itself, the group a dense FFN and reduce_moe_output reduce over.""" parallel = get_parallel() if parallel.moe_ep_size > 1 and parallel.moe_tp_size > 1: # Some MoE blocks reduce EP and MoE-TP in two steps instead of merging. return False - group = parallel.moe_ep_group if parallel.moe_ep_size > 1 else parallel.moe_tp_group - return group is parallel.tp_group + return post_experts_reduction_group() is parallel.tp_group def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool: @@ -684,8 +685,8 @@ def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool: def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: - """Whether deferred_post_experts_all_reduce is the all-reduce the FFN would - have run itself: one full-precision sum over the same TP group, on an output + """Whether the next layer's all-reduce is the one the FFN would have run + itself: one full-precision sum over the same TP group, on an output that is a plain partial sum. The output is not a plain partial sum when the MoE combine already summed it, a replicated shared expert is added after the reduction, or LoRA-B runs on the unreduced activations.""" @@ -729,6 +730,13 @@ class LayerCommunicator: force_layernorm_before_dp_gather ) self._post_init_communicate() + # Under attention DP, the base postprocess scatters the FFN output back + # to this rank's tokens, which the next layer's input can run instead; + # the MHC and DSA-CP postprocess do more, so theirs stays here. + self._postprocess_scatters_to_local_tokens = ( + self._communicate_summable_tensor_pair_fn + is CommunicateSummableTensorPairFn._scatter_hidden_states + ) self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm ) @@ -855,9 +863,12 @@ class LayerCommunicator: # No fused kernel runs under attention DP: the reduce-scatter back to # this rank's tokens comes first. hidden_states = reduce_output(hidden_states) - pending = isinstance(hidden_states, UnreducedOutput) + unreduced = ( + hidden_states if isinstance(hidden_states, UnreducedOutput) else None + ) + pending = unreduced is not None if pending: - hidden_states = hidden_states.partial + hidden_states = unreduced.partial # residual is None marks the first decoder layer, where the SP region # opens: re-evaluated per forward so a crash mid-loop cannot leak into # the next one. @@ -889,12 +900,14 @@ class LayerCommunicator: hidden_states, residual, forward_batch ) if post_residual_addition is None + # The fused kernel reduces over the MoE output's group. + and unreduced.group is post_experts_reduction_group() else None ) if fused is not None: hidden_states, residual = fused else: - hidden_states = deferred_post_experts_all_reduce(hidden_states) + hidden_states = reduce_output(unreduced) hidden_states, residual = _attn_input_update_and_read_residual( quant_format )(self.input_layernorm, hidden_states, residual, post_residual_addition) @@ -1016,27 +1029,74 @@ class LayerCommunicator: is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) - def _reduce_scatter_for_next_layer( + def _local_token_move_can_go_to_next_layer(self) -> bool: + """Whether the next layer's input can run this layer's move of its FFN + output back to this rank's tokens: the base postprocess scatter, outside + an active LayerNorm SP region.""" + return self._postprocess_scatters_to_local_tokens and not ( + self._sp_variant is not None and get_forward().sp_active + ) + + def _reduce_scatter_step( self, forward_batch: ForwardBatch - ) -> Optional[Callable[[torch.Tensor], torch.Tensor]]: - """The reduce-scatter back to this rank's tokens that postprocess would - run on this layer's FFN output, bound for the next layer's input to run - instead; None when postprocess runs anything else.""" - if ( - self.is_last_layer - or (self._sp_variant is not None and get_forward().sp_active) - or self._communicate_summable_tensor_pair_fn - is not CommunicateSummableTensorPairFn._scatter_hidden_states - ): - return None - step = _output_to_local_tokens_step( + ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: + """The reduce-scatter postprocess would run on this layer's FFN output; + None when it runs only a scatter, or anything else.""" + return _reduce_and_redistribute_output_step( forward_batch, allow_reduce_scatter=self.allow_reduce_scatter, is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) - if step is _redistribute_output: - return None - return partial(_to_local_tokens, step, forward_batch) + + def ffn_reduction_group(self) -> GroupCoordinator: + """The group this layer's FFN output owes its sum over: the MoE output's + group on a sparse layer, the TP group a dense MLP reduces over.""" + if self.layer_scatter_modes.is_layer_sparse: + return post_experts_reduction_group() + return get_parallel().tp_group + + def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": + """Decide once, before the FFN runs, what it skips and what completes its + output: the next layer's input, or this layer's postprocess.""" + defer_moe_finalize = self.should_defer_moe_finalize(forward_batch) + # Deferring implies fusing: a handoff skips the post-experts all-reduce. + fuse_mlp_allreduce = defer_moe_finalize or self.should_defer_ffn_reduction( + forward_batch + ) + mlp_reduce_scatter = self.should_use_reduce_scatter(forward_batch) + if fuse_mlp_allreduce: + group = self.ffn_reduction_group() + if self._postprocess_scatters_to_local_tokens: + # Under attention DP the next layer also brings the sum back to + # this rank's tokens. + leave = partial( + UnreducedOutput, + reduce_and_redistribute=partial( + _all_reduce_then_to_local_tokens, group, forward_batch + ), + ) + else: + leave = partial(UnreducedOutput, group=group) + elif not self.is_last_layer and self._local_token_move_can_go_to_next_layer(): + step = self._reduce_scatter_step(forward_batch) + leave = ( + None + if step is None + else partial( + UnreducedOutput, + reduce_and_redistribute=partial( + _to_local_tokens, step, forward_batch + ), + ) + ) + else: + leave = None + return FfnCompletion( + defer_moe_finalize=defer_moe_finalize, + fuse_mlp_allreduce=fuse_mlp_allreduce, + mlp_reduce_scatter=mlp_reduce_scatter, + leave=leave, + ) def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": """Decide once how this layer's FFN output reduction completes. Use the @@ -1155,41 +1215,17 @@ class LayerCommunicator: and self._ffn_sum_can_move_to_next_layer() and _unfused_completion_matches_the_ffn(forward_batch) and not self.should_use_reduce_scatter(forward_batch) - and self._next_layer_input_can_restore_the_layout(forward_batch) - ) - - def _next_layer_input_can_restore_the_layout( - self, forward_batch: ForwardBatch - ) -> bool: - """Whether the next layer's input can also run what postprocess would - after the FFN's own all-reduce: under attention DP, the scatter back to - this rank's tokens.""" - if not is_dp_attention_enabled(): - return True - return ( - self._communicate_summable_tensor_pair_fn - is CommunicateSummableTensorPairFn._scatter_hidden_states - and not (self._sp_variant is not None and get_forward().sp_active) - and _output_to_local_tokens_step( - forward_batch, - allow_reduce_scatter=self.allow_reduce_scatter, - is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + # Under attention DP the next layer must also run postprocess's + # scatter back to this rank's tokens, and nothing more. + and ( + not is_dp_attention_enabled() + or ( + self._local_token_move_can_go_to_next_layer() + and self._reduce_scatter_step(forward_batch) is None + ) ) - is _redistribute_output ) - def _scatter_for_next_layer( - self, forward_batch: ForwardBatch - ) -> Optional[Callable[[torch.Tensor], torch.Tensor]]: - """Under attention DP, the all-reduce and the scatter back to this rank's - tokens that the next layer's input runs on a sum this layer's FFN left.""" - if ( - self._communicate_summable_tensor_pair_fn - is not CommunicateSummableTensorPairFn._scatter_hidden_states - ): - return None - return partial(_all_reduce_then_to_local_tokens, forward_batch) - # MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on. _SCATTER_MODE_SHARDED_AXES = { @@ -1218,10 +1254,21 @@ def scatter_mode_layouts( } +class FfnCompletion(msgspec.Struct, frozen=True): + """One FFN's reduction decision. The flags are published while the FFN runs; + ``leave`` wraps its output for the next layer's input to complete, or is None + when this layer's postprocess completes it.""" + + defer_moe_finalize: bool + fuse_mlp_allreduce: bool + mlp_reduce_scatter: bool + leave: Optional[Callable[[torch.Tensor], UnreducedOutput]] = None + + class FfnExit: - """One FFN's reduction decision. Inside the ``with`` block it is published as - ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / ``defer_moe_finalize`` on - ``get_forward()``.""" + """The scope that publishes an FfnCompletion while the FFN runs: inside the + ``with`` block it is ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` / + ``defer_moe_finalize`` on ``get_forward()``.""" __slots__ = ( "communicator", @@ -1229,19 +1276,18 @@ class FfnExit: "defer_moe_finalize", "fuse_mlp_allreduce", "mlp_reduce_scatter", + "_leave", "_scope", ) def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): self.communicator = communicator self.forward_batch = forward_batch - self.defer_moe_finalize = communicator.should_defer_moe_finalize(forward_batch) - # Deferring implies fusing: a handoff skips the post-experts all-reduce. - self.fuse_mlp_allreduce = ( - self.defer_moe_finalize - or communicator.should_defer_ffn_reduction(forward_batch) - ) - self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch) + completion = communicator._select_ffn_completion(forward_batch) + self.defer_moe_finalize = completion.defer_moe_finalize + self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce + self.mlp_reduce_scatter = completion.mlp_reduce_scatter + self._leave = completion.leave self._scope = get_forward().scoped( fuse_mlp_allreduce=self.fuse_mlp_allreduce, mlp_reduce_scatter=self.mlp_reduce_scatter, @@ -1263,26 +1309,8 @@ class FfnExit: # A deferred MoE finalize handoff, consumed by the next prepare_attn. assert self.defer_moe_finalize, "unrequested deferred MoE handoff" return hidden_states, residual - if self.fuse_mlp_allreduce: - return ( - UnreducedOutput( - hidden_states, - reduce_and_redistribute=self.communicator._scatter_for_next_layer( - self.forward_batch - ), - ), - residual, - ) - reduce_and_redistribute = self.communicator._reduce_scatter_for_next_layer( - self.forward_batch - ) - if reduce_and_redistribute is not None: - return ( - UnreducedOutput( - hidden_states, reduce_and_redistribute=reduce_and_redistribute - ), - residual, - ) + if self._leave is not None: + return self._leave(hidden_states), residual return self.communicator.postprocess_layer( hidden_states, residual, self.forward_batch ) @@ -1797,12 +1825,12 @@ def _redistribute_output_from_moe_cp( ).contiguous() -def _output_to_local_tokens_step( +def _reduce_and_redistribute_output_step( forward_batch: ForwardBatch, *, allow_reduce_scatter: bool, is_layer_sparse: bool -) -> Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]: - """How a FULL-layout layer output comes back to this rank's tokens under - attention DP: a reduce-scatter when the FFN left its sum to it, otherwise a - scatter of the reduced output.""" +) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: + """The reduce-scatter that brings a FULL-layout layer output back to this + rank's tokens under attention DP when the FFN left its sum to it; None when + the FFN reduced the output and only a scatter remains.""" # A MoE block leaves its sum to reduce_scatterv whenever it applies # (should_skip_post_experts_all_reduce); a dense MLP does only under the # published mlp_reduce_scatter, which needs allow_reduce_scatter. @@ -1814,7 +1842,7 @@ def _output_to_local_tokens_step( and can_use_dp_reduce_scatter() ): return _reduce_and_redistribute_output_max_len - return _redistribute_output + return None def _to_local_tokens( @@ -1828,12 +1856,10 @@ def _to_local_tokens( def _all_reduce_then_to_local_tokens( - forward_batch: ForwardBatch, hidden_states: torch.Tensor + group: GroupCoordinator, forward_batch: ForwardBatch, hidden_states: torch.Tensor ) -> torch.Tensor: return _to_local_tokens( - _redistribute_output, - forward_batch, - deferred_post_experts_all_reduce(hidden_states), + _redistribute_output, forward_batch, group.all_reduce(hidden_states) ) @@ -1917,10 +1943,13 @@ class CommunicateSummableTensorPairFn: is_layer_sparse: bool = False, ): local_hidden_states = get_local_dp_buffer(_dp_scatter_group()) - step = _output_to_local_tokens_step( - forward_batch, - allow_reduce_scatter=allow_reduce_scatter, - is_layer_sparse=is_layer_sparse, + step = ( + _reduce_and_redistribute_output_step( + forward_batch, + allow_reduce_scatter=allow_reduce_scatter, + is_layer_sparse=is_layer_sparse, + ) + or _redistribute_output ) step(local_hidden_states, hidden_states, forward_batch) return local_hidden_states, residual diff --git a/python/sglang/srt/layers/flashinfer_comm_fusion.py b/python/sglang/srt/layers/flashinfer_comm_fusion.py index d7b59408b2..2cf739d083 100644 --- a/python/sglang/srt/layers/flashinfer_comm_fusion.py +++ b/python/sglang/srt/layers/flashinfer_comm_fusion.py @@ -673,16 +673,13 @@ def resolve_fusion_group(*, use_attn_tp_group: bool): Must match the group the fused residual+LN kernel reduces over; a mismatch silently reduces across the wrong peers. """ - from sglang.srt.layers.moe.utils import can_merge_post_experts_all_reduce + from sglang.srt.layers.moe.utils import post_experts_reduction_group parallel = get_parallel() if use_attn_tp_group: return parallel.attn_tp_size, parallel.attn_tp_rank, parallel.attn_tp_group - if can_merge_post_experts_all_reduce(): - return parallel.tp_size, parallel.tp_rank, parallel.tp_group - if parallel.moe_ep_size > 1: - return parallel.moe_ep_size, parallel.moe_ep_rank, parallel.moe_ep_group - return parallel.moe_tp_size, parallel.moe_tp_rank, parallel.moe_tp_group + group = post_experts_reduction_group() + return group.world_size, group.rank_in_group, group def _sync_allreduce_unavailable_across_tp(): diff --git a/python/sglang/srt/layers/moe/__init__.py b/python/sglang/srt/layers/moe/__init__.py index 9fcb7c6c95..17c8b3165b 100644 --- a/python/sglang/srt/layers/moe/__init__.py +++ b/python/sglang/srt/layers/moe/__init__.py @@ -15,6 +15,7 @@ from sglang.srt.layers.moe.utils import ( is_tbo_enabled, post_experts_all_reduce, post_experts_output_is_complete, + post_experts_reduction_group, reduce_moe_output, should_skip_mlp_all_reduce, should_skip_post_experts_all_reduce, @@ -37,6 +38,7 @@ __all__ = [ "deferred_post_experts_all_reduce", "post_experts_all_reduce", "post_experts_output_is_complete", + "post_experts_reduction_group", "reduce_moe_output", "should_skip_post_experts_all_reduce", "should_use_dp_reduce_scatterv", diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 2ef21189b1..32b5f4d340 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -838,23 +838,24 @@ def post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states +def post_experts_reduction_group(): + """The group one all-reduce of an MoE output runs over: TP when the EP and + MoE-TP reductions merge, otherwise EP, otherwise MoE-TP. The same group + ``resolve_fusion_group`` builds the fused workspace on.""" + parallel = get_parallel() + if can_merge_post_experts_all_reduce(): + return parallel.tp_group + if parallel.moe_ep_size > 1: + return parallel.moe_ep_group + return parallel.moe_tp_group + + def deferred_post_experts_all_reduce(hidden_states: torch.Tensor) -> torch.Tensor: """Run the post-experts reduction that was deferred to allreduce fusion. - Called when the fused residual+LN kernel cannot service the shape. Reduces - over the same group ``resolve_fusion_group`` builds the workspace on. + Called when the fused residual+LN kernel cannot service the shape. """ - from sglang.srt.distributed.communication_op import ( - moe_expert_parallel_all_reduce, - moe_tensor_model_parallel_all_reduce, - tensor_model_parallel_all_reduce, - ) - - if can_merge_post_experts_all_reduce(): - return tensor_model_parallel_all_reduce(hidden_states) - if get_parallel().moe_ep_size > 1: - return moe_expert_parallel_all_reduce(hidden_states) - return moe_tensor_model_parallel_all_reduce(hidden_states) + return post_experts_reduction_group().all_reduce(hidden_states) @contextmanager diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 4f8ac807bb..837ebce7ad 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -427,7 +427,9 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): ): hidden_states = self.mixer.forward(hidden_states) if fuse_mlp_allreduce: - hidden_states = UnreducedOutput(hidden_states) + hidden_states = UnreducedOutput( + hidden_states, group=self.layer_communicator.ffn_reduction_group() + ) else: hidden_states, residual = self.layer_communicator.postprocess_layer( hidden_states, residual, forward_batch @@ -531,7 +533,9 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): hidden_states, forward_batch, skip_reduce ) if fuse_mlp_allreduce: - hidden_states = UnreducedOutput(hidden_states) + hidden_states = UnreducedOutput( + hidden_states, group=get_parallel().tp_group + ) return hidden_states, residual diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 43353cb132..3fdfc56e6a 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -9,7 +9,6 @@ import torch import sglang from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( - CommunicateSummableTensorPairFn, LayerCommunicator, UnreducedOutput, reduce_output, @@ -21,12 +20,18 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") +def make_group(scale=3): + """The group a deferred sum is owed over, with a stubbed all-reduce.""" + return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale)) + + def make_communicator( *, fuse, reduce_scatter, - next_layer_reduce_scatter=None, - next_layer_scatter=None, + reduce_scatter_step=None, + scatters_to_local_tokens=False, + group=None, cls=LayerCommunicator, ): """A communicator whose decisions and postprocess are stubbed, built @@ -35,10 +40,11 @@ def make_communicator( if cls is LayerCommunicator: communicator.should_defer_ffn_reduction = MagicMock(return_value=fuse) communicator.should_use_reduce_scatter = MagicMock(return_value=reduce_scatter) - communicator._reduce_scatter_for_next_layer = MagicMock( - return_value=next_layer_reduce_scatter - ) - communicator._scatter_for_next_layer = MagicMock(return_value=next_layer_scatter) + communicator.is_last_layer = False + communicator._sp_variant = None + communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens + communicator._reduce_scatter_step = MagicMock(return_value=reduce_scatter_step) + communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) communicator.postprocess_layer = MagicMock( side_effect=lambda hidden_states, residual, forward_batch: ( hidden_states + 1, @@ -73,31 +79,46 @@ class TestFfnExit(CustomTestCase): self.assertIs(residual, self.residual) communicator.postprocess_layer.assert_not_called() + def test_reduction_left_to_next_layer_declares_its_group(self): + group = make_group() + communicator = make_communicator(fuse=True, reduce_scatter=False, group=group) + _, (hidden_states, _) = self.run_exit(communicator) + self.assertIs(hidden_states.group, group) + self.assertIsNone(hidden_states.reduce_and_redistribute) + def test_a_reduce_scatter_is_left_to_the_next_layer(self): """Under attention DP the reduce-scatter postprocess would run goes to the next layer with the partial sum.""" step = MagicMock() communicator = make_communicator( - fuse=False, reduce_scatter=True, next_layer_reduce_scatter=step + fuse=False, + reduce_scatter=True, + reduce_scatter_step=step, + scatters_to_local_tokens=True, ) seen, (hidden_states, residual) = self.run_exit(communicator) self.assertEqual(seen, (False, True)) self.assertIsInstance(hidden_states, UnreducedOutput) - self.assertIs(hidden_states.reduce_and_redistribute, step) + bound = hidden_states.reduce_and_redistribute + self.assertIs(bound.func, comm._to_local_tokens) + self.assertEqual(bound.args, (step, self.forward_batch)) self.assertIs(residual, self.residual) communicator.postprocess_layer.assert_not_called() step.assert_not_called() def test_a_deferred_sum_carries_the_scatter_back_under_attention_dp(self): - step = MagicMock() + group = make_group() communicator = make_communicator( - fuse=True, reduce_scatter=False, next_layer_scatter=step + fuse=True, reduce_scatter=False, scatters_to_local_tokens=True, group=group ) _, (hidden_states, _) = self.run_exit(communicator) self.assertIsInstance(hidden_states, UnreducedOutput) - self.assertIs(hidden_states.reduce_and_redistribute, step) - communicator._reduce_scatter_for_next_layer.assert_not_called() + bound = hidden_states.reduce_and_redistribute + self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) + self.assertEqual(bound.args, (group, self.forward_batch)) + communicator._reduce_scatter_step.assert_not_called() communicator.postprocess_layer.assert_not_called() + group.all_reduce.assert_not_called() def test_postprocess_completes_other_exits(self): for reduce_scatter in (False, True): @@ -174,16 +195,14 @@ class TestFfnExit(CustomTestCase): class TestReduceOutput(CustomTestCase): def setUp(self): - self.all_reduce = MagicMock(side_effect=lambda hidden_states: hidden_states * 3) - patcher = patch( - "sglang.srt.layers.communicator.deferred_post_experts_all_reduce", - self.all_reduce, - ) - patcher.start() - self.addCleanup(patcher.stop) + self.group = make_group() + self.all_reduce = self.group.all_reduce + + def communicator(self, *, fuse): + return make_communicator(fuse=fuse, reduce_scatter=False, group=self.group) def test_reduction_left_by_the_last_layer_runs_once(self): - communicator = make_communicator(fuse=True, reduce_scatter=False) + communicator = self.communicator(fuse=True) with communicator.ffn_exit(object()) as ffn_exit: hidden_states = torch.ones(3, 4) hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) @@ -208,7 +227,7 @@ class TestReduceOutput(CustomTestCase): self.all_reduce.assert_not_called() def test_complete_hidden_states_pass_through(self): - communicator = make_communicator(fuse=False, reduce_scatter=False) + communicator = self.communicator(fuse=False) with communicator.ffn_exit(object()) as ffn_exit: hidden_states = torch.ones(3, 4) hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) @@ -221,7 +240,7 @@ class TestReduceOutput(CustomTestCase): for fuse in (True, False): with self.subTest(fuse=fuse): self.all_reduce.reset_mock() - communicator = make_communicator(fuse=fuse, reduce_scatter=False) + communicator = self.communicator(fuse=fuse) residual = torch.zeros(3, 4) with communicator.ffn_exit(object()) as ffn_exit: hidden_states = torch.ones(3, 4) @@ -236,87 +255,82 @@ class TestReduceOutput(CustomTestCase): self.assertIs(residual_out, residual) -class TestReduceScatterForNextLayer(CustomTestCase): - """Which reduce-scatter the next layer runs in place of postprocess.""" +class TestSelectFfnCompletion(CustomTestCase): + """What the next layer's input runs in place of postprocess, chosen before + the FFN runs.""" - def communicator(self, *, is_last_layer=False, pair_fn=None, sp_variant=None): + def communicator( + self, *, fuse=False, is_last_layer=False, scatters=True, sp_variant=None + ): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator.is_last_layer = is_last_layer communicator._sp_variant = sp_variant - communicator._communicate_summable_tensor_pair_fn = ( - pair_fn or CommunicateSummableTensorPairFn._scatter_hidden_states - ) + communicator._postprocess_scatters_to_local_tokens = scatters communicator.allow_reduce_scatter = True communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) + communicator.should_defer_ffn_reduction = lambda forward_batch: fuse + communicator.should_use_reduce_scatter = lambda forward_batch: not fuse + self.group = make_group() + communicator.ffn_reduction_group = lambda: self.group return communicator - def bound(self, communicator, step): - with patch.object(comm, "_output_to_local_tokens_step", return_value=step): - return communicator._reduce_scatter_for_next_layer(object()) + def left(self, communicator, step, forward_batch=None): + with patch.object( + comm, "_reduce_and_redistribute_output_step", return_value=step + ): + completion = communicator._select_ffn_completion(forward_batch) + if completion.leave is None: + return None + return completion.leave(torch.ones(3, 4)) def test_a_reduce_scatter_is_bound_for_the_next_layer(self): + forward_batch = object() for step in ( comm._reduce_and_redistribute_output_varlen, comm._reduce_and_redistribute_output_max_len, ): with self.subTest(step=step.__name__): - bound = self.bound(self.communicator(), step) + left = self.left(self.communicator(), step, forward_batch) + bound = left.reduce_and_redistribute self.assertIs(bound.func, comm._to_local_tokens) - self.assertIs(bound.args[0], step) + self.assertEqual(bound.args, (step, forward_batch)) def test_postprocess_keeps_everything_else(self): reduce_scatter = comm._reduce_and_redistribute_output_varlen for name, communicator, step in ( - ("scatter only", self.communicator(), comm._redistribute_output), + ("scatter only", self.communicator(), None), ("last layer", self.communicator(is_last_layer=True), reduce_scatter), - ( - "other layout change", - self.communicator( - pair_fn=CommunicateSummableTensorPairFn._scatter_hidden_states_moe - ), - reduce_scatter, - ), + ("other postprocess", self.communicator(scatters=False), reduce_scatter), ): with self.subTest(name): - self.assertIsNone(self.bound(communicator, step)) + self.assertIsNone(self.left(communicator, step)) def test_postprocess_keeps_an_active_layernorm_sp_region(self): communicator = self.communicator(sp_variant=object()) with get_forward().scoped(sp_active=True): self.assertIsNone( - self.bound(communicator, comm._reduce_and_redistribute_output_varlen) + self.left(communicator, comm._reduce_and_redistribute_output_varlen) ) - -class TestScatterForNextLayer(CustomTestCase): - def communicator(self, pair_fn): - communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator._communicate_summable_tensor_pair_fn = pair_fn - return communicator - - def test_only_a_scatter_back_to_local_tokens_is_carried(self): + def test_a_deferred_sum_keeps_its_layout_or_scatters_back(self): forward_batch = object() - bound = self.communicator( - CommunicateSummableTensorPairFn._scatter_hidden_states - )._scatter_for_next_layer(forward_batch) + kept = self.left(self.communicator(fuse=True, scatters=False), None) + self.assertIs(kept.group, self.group) + self.assertIsNone(kept.reduce_and_redistribute) + moved = self.left(self.communicator(fuse=True), None, forward_batch) + self.assertIsNone(moved.group) + bound = moved.reduce_and_redistribute self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) - self.assertIs(bound.args[0], forward_batch) - self.assertIsNone( - self.communicator( - CommunicateSummableTensorPairFn._trivial - )._scatter_for_next_layer(forward_batch) - ) + self.assertEqual(bound.args, (self.group, forward_batch)) def test_the_all_reduce_runs_before_the_scatter(self): calls = [] partial = torch.ones(3, 4) local = torch.empty(1, 4) + group = types.SimpleNamespace( + all_reduce=lambda x: calls.append(("all_reduce", x)) or x * 2 + ) with ( - patch.object( - comm, - "deferred_post_experts_all_reduce", - side_effect=lambda x: calls.append(("all_reduce", x)) or x * 2, - ), patch.object(comm, "_dp_scatter_group", return_value="group"), patch.object( comm, @@ -329,7 +343,7 @@ class TestScatterForNextLayer(CustomTestCase): side_effect=lambda out, full, fb: calls.append(("scatter", out)), ), ): - result = comm._all_reduce_then_to_local_tokens(object(), partial) + result = comm._all_reduce_then_to_local_tokens(group, object(), partial) self.assertEqual([c[0] for c in calls], ["all_reduce", "buffer", "scatter"]) self.assertIs(calls[0][1], partial) self.assertIs(result, local) diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 3712e177f3..0d1ed19c96 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -112,12 +112,19 @@ class TestDeferredPostExpertsAllReduce(CustomTestCase): def _calls(self, *, moe_ep_size, moe_tp_size, moe_dp_size=1): called = [] - with _recorded_all_reduces( - called, + + def group(name): + return types.SimpleNamespace(all_reduce=lambda x: called.append(name) or x) + + parallel = types.SimpleNamespace( moe_ep_size=moe_ep_size, moe_tp_size=moe_tp_size, moe_dp_size=moe_dp_size, - ): + tp_group=group("tp"), + moe_ep_group=group("ep"), + moe_tp_group=group("moe_tp"), + ) + with patch.object(moe_utils, "get_parallel", return_value=parallel): deferred_post_experts_all_reduce(torch.zeros(2, 2)) return called @@ -164,10 +171,20 @@ class TestResolveFusionGroup(CustomTestCase): resolve_fusion_world_size, ) - fake_tp_group = MagicMock(name="tp_group") - fake_ep_group = MagicMock(name="ep_group") - fake_moe_tp_group = MagicMock(name="moe_tp_group") tp_size = moe_ep_size * moe_tp_size * moe_dp_size + fake_tp_group = MagicMock( + name="tp_group", world_size=tp_size, rank_in_group=tp_rank + ) + fake_ep_group = MagicMock( + name="ep_group", + world_size=moe_ep_size, + rank_in_group=tp_rank % moe_ep_size, + ) + fake_moe_tp_group = MagicMock( + name="moe_tp_group", + world_size=moe_tp_size, + rank_in_group=tp_rank % moe_tp_size, + ) with get_parallel().override( moe_ep_size=moe_ep_size, @@ -299,14 +316,14 @@ class TestDeferFfnReduction(CustomTestCase): lora=False, tp_group=True, global_tokens=8, - pair_fn=comm.CommunicateSummableTensorPairFn._scatter_hidden_states, - step=comm._redistribute_output, + scatters_to_local_tokens=True, + step=None, sp_active=False, ): communicator = _fake_communicator() communicator.is_last_layer = is_last_layer communicator.should_use_reduce_scatter = lambda forward_batch: reduce_scatter - communicator._communicate_summable_tensor_pair_fn = pair_fn + communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._sp_variant = object() if sp_active else None communicator.allow_reduce_scatter = True communicator.layer_scatter_modes.is_layer_sparse = True @@ -315,7 +332,9 @@ class TestDeferFfnReduction(CustomTestCase): global_dp_buffer_len=global_tokens, ) with ( - patch.object(comm, "_output_to_local_tokens_step", return_value=step), + patch.object( + comm, "_reduce_and_redistribute_output_step", return_value=step + ), get_forward().scoped(sp_active=sp_active), patch.object(comm, "is_enable_moe_cp_allgather", return_value=False), patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fused), @@ -404,12 +423,7 @@ class TestDeferFfnReductionUnderAttentionDp(CustomTestCase): "MAX_LEN reduce-scatter", dict(step=comm._reduce_and_redistribute_output_max_len), ), - ( - "other layout change", - dict( - pair_fn=comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe - ), - ), + ("other postprocess", dict(scatters_to_local_tokens=False)), ("LayerNorm SP", dict(sp_active=True)), ): with self.subTest(name): @@ -427,11 +441,15 @@ class TestDeferredReductionGroup(CustomTestCase): parallel = types.SimpleNamespace( moe_ep_size=moe_ep_size, moe_tp_size=moe_tp_size, + moe_dp_size=1, tp_group=tp_group, moe_ep_group=tp_group if ep_is_tp else object(), moe_tp_group=tp_group if moe_tp_is_tp else object(), ) - with patch.object(comm, "get_parallel", return_value=parallel): + with ( + patch.object(comm, "get_parallel", return_value=parallel), + patch.object(moe_utils, "get_parallel", return_value=parallel), + ): return comm._deferred_reduction_runs_on_the_tp_group() def test_pure_tp_and_pure_ep_reduce_over_the_tp_group(self): diff --git a/test/registered/unit/layers/test_layernorm_allreduce_fusion.py b/test/registered/unit/layers/test_layernorm_allreduce_fusion.py index 33c4116582..f99214aa4b 100644 --- a/test/registered/unit/layers/test_layernorm_allreduce_fusion.py +++ b/test/registered/unit/layers/test_layernorm_allreduce_fusion.py @@ -3,6 +3,7 @@ batch, forward_with_allreduce_fusion still completes the sum, over the group the kernel reduces over.""" import contextlib +import types import unittest from unittest.mock import patch @@ -11,6 +12,7 @@ import torch from sglang.srt.distributed import communication_op from sglang.srt.layers import flashinfer_comm_fusion, layernorm from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -55,6 +57,21 @@ def declined_fused_kernel(reduced, *, moe_ep_size, moe_tp_size): communication_op, getter, return_value=Group(name, reduced) ) ) + # The MoE output's group is read from the parallel state directly. + stack.enter_context( + patch.object( + moe_utils, + "get_parallel", + return_value=types.SimpleNamespace( + moe_ep_size=moe_ep_size, + moe_tp_size=moe_tp_size, + moe_dp_size=1, + tp_group=Group("tp", reduced), + moe_ep_group=Group("moe_ep", reduced), + moe_tp_group=Group("moe_tp", reduced), + ), + ) + ) stack.enter_context(patch.object(layernorm, "_use_aiter", False)) stack.enter_context( patch.object( diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 36a9535b74..f0d57837c5 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -151,8 +151,9 @@ class TestPostprocessReduceScatterv(CustomTestCase): self.assertIs(seen["is_layer_sparse"], True) -class TestOutputToLocalTokensStep(CustomTestCase): - """The step that brings a FULL-layout output back to this rank's tokens.""" +class TestReduceAndRedistributeOutputStep(CustomTestCase): + """The reduce-scatter that brings a FULL-layout output back to this rank's + tokens, or None when only a scatter remains.""" def step(self, *, varlen, max_len, tiles, allow, sparse): forward_batch = types.SimpleNamespace( @@ -162,7 +163,7 @@ class TestOutputToLocalTokensStep(CustomTestCase): patch.object(comm, "should_use_dp_reduce_scatterv", return_value=varlen), patch.object(comm, "can_use_dp_reduce_scatter", return_value=tiles), ): - return comm._output_to_local_tokens_step( + return comm._reduce_and_redistribute_output_step( forward_batch, allow_reduce_scatter=allow, is_layer_sparse=sparse ) @@ -175,7 +176,7 @@ class TestOutputToLocalTokensStep(CustomTestCase): elif allow and max_len and tiles: expected = comm._reduce_and_redistribute_output_max_len else: - expected = comm._redistribute_output + expected = None case = dict( varlen=varlen, max_len=max_len, tiles=tiles, allow=allow, sparse=sparse ) diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 71a9d5cb87..f6c2f433ef 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -52,12 +52,14 @@ def communicator(norm): @contextlib.contextmanager -def platform(*, use_aiter=False, gfx95=False, fusion=False): +def platform(*, use_aiter=False, gfx95=False, fusion=False, kernel_group=True): kernels = {name: MagicMock(name=name) for name in KERNELS} kernels["fused_rms_mxfp4_quant"].return_value = ("q", None, None, "r") kernels["fused_rms_fp8_group_quant"].return_value = (("q", "s"), None, None, "r") kernels["_fused_rmsnorm_fp8_per_token_quant"].return_value = ("q", "r") all_reduce = MagicMock(side_effect=lambda h: h * 5) + # The group an unreduced output owes its sum over. + group = SimpleNamespace(all_reduce=all_reduce) with contextlib.ExitStack() as stack: for name, mock in kernels.items(): stack.enter_context(patch.object(comm, name, mock, create=True)) @@ -72,8 +74,13 @@ def platform(*, use_aiter=False, gfx95=False, fusion=False): stack.enter_context( patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fusion) ) + # The group the fused kernel reduces over. stack.enter_context( - patch.object(comm, "deferred_post_experts_all_reduce", all_reduce) + patch.object( + comm, + "post_experts_reduction_group", + return_value=group if kernel_group else SimpleNamespace(), + ) ) stack.enter_context( patch.object( @@ -82,7 +89,7 @@ def platform(*, use_aiter=False, gfx95=False, fusion=False): return_value=SimpleNamespace(input_scattered=False, is_dsa=False), ) ) - yield kernels, all_reduce + yield kernels, all_reduce, group class TestPrepareAttnSteps(CustomTestCase): @@ -105,7 +112,7 @@ class TestPrepareAttnSteps(CustomTestCase): flags=flags, residual=residual is not None, ), - platform(**flags) as (kernels, _), + platform(**flags) as (kernels, _, _), ): norm = Norm() communicator(norm).prepare_attn( @@ -133,10 +140,10 @@ class TestPrepareAttnSteps(CustomTestCase): for fusion in (False, True): with ( self.subTest(fusion=fusion), - platform(fusion=fusion) as (_, all_reduce), + platform(fusion=fusion) as (_, all_reduce, group), ): norm = Norm() - hidden_states = comm.UnreducedOutput(torch.ones(2, 4)) + hidden_states = comm.UnreducedOutput(torch.ones(2, 4), group=group) communicator(norm).prepare_attn(hidden_states, torch.zeros(2, 4), None) self.assertEqual(all_reduce.call_count, 0 if fusion else 1) self.assertEqual( @@ -144,10 +151,10 @@ class TestPrepareAttnSteps(CustomTestCase): ) def test_a_pending_sum_keeps_the_quant_format_without_fusion(self): - with platform(use_aiter=True) as (kernels, all_reduce): + with platform(use_aiter=True) as (kernels, all_reduce, group): norm = Norm() communicator(norm).prepare_attn( - comm.UnreducedOutput(torch.ones(2, 4)), + comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, quant_format="fp8_per_token", @@ -156,11 +163,22 @@ class TestPrepareAttnSteps(CustomTestCase): self.assertTrue(kernels["_fused_rmsnorm_fp8_per_token_quant"].called) self.assertEqual(norm.calls, []) - def test_a_post_residual_addition_bypasses_the_fused_kernel(self): - with platform(fusion=True) as (_, all_reduce): + def test_a_pending_sum_over_another_group_skips_the_fused_kernel(self): + with platform(fusion=True, kernel_group=False) as (_, all_reduce, group): norm = Norm() communicator(norm).prepare_attn( - comm.UnreducedOutput(torch.ones(2, 4)), + comm.UnreducedOutput(torch.ones(2, 4), group=group), + torch.zeros(2, 4), + None, + ) + self.assertEqual(all_reduce.call_count, 1) + self.assertEqual(norm.calls, ["norm"]) + + def test_a_post_residual_addition_bypasses_the_fused_kernel(self): + with platform(fusion=True) as (_, all_reduce, group): + norm = Norm() + communicator(norm).prepare_attn( + comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, post_residual_addition=torch.ones(2, 4), @@ -172,7 +190,7 @@ class TestPrepareAttnSteps(CustomTestCase): for local_rows in (1, 0): with ( self.subTest(local_rows=local_rows), - platform(fusion=True) as (_, all_reduce), + platform(fusion=True) as (_, all_reduce, _), ): norm = Norm() partial = torch.ones(3, 4) @@ -196,7 +214,7 @@ class TestPrepareAttnSteps(CustomTestCase): for step in (None, MagicMock()): with ( self.subTest(redistributes=step is not None), - platform(fusion=True) as (_, all_reduce), + platform(fusion=True) as (_, all_reduce, _), ): norm = Norm() with self.assertRaises(RuntimeError): diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 9a9871b3ab..067c163af4 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -37,6 +37,10 @@ def all_reduce(hidden_states): return hidden_states * 2 +# The group a deferred FFN output owes its sum over. +GROUP = SimpleNamespace(all_reduce=all_reduce) + + class DeferringLayer(nn.Module): def __init__(self, defer, return_topk=False): super().__init__() @@ -46,8 +50,9 @@ class DeferringLayer(nn.Module): lambda batch: defer ) self.layer_communicator.should_defer_ffn_reduction = lambda batch: defer - self.layer_communicator._reduce_scatter_for_next_layer = lambda batch: None - self.layer_communicator._scatter_for_next_layer = lambda batch: None + self.layer_communicator.is_last_layer = False + self.layer_communicator._postprocess_scatters_to_local_tokens = False + self.layer_communicator.ffn_reduction_group = lambda: GROUP self.layer_communicator.should_use_reduce_scatter = lambda batch: False self.layer_communicator.postprocess_layer = lambda hidden, residual, batch: ( all_reduce(hidden), @@ -126,9 +131,7 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): model=model_cls.__name__, defer=defer, capture=capture ), patch.object( - comm, - "deferred_post_experts_all_reduce", - side_effect=all_reduce, + GROUP, "all_reduce", side_effect=all_reduce ) as reduce, ): model = build_model(model_cls, defer=defer, capture=capture) diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index 36313747d8..02af8943ce 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -4,7 +4,6 @@ partial sum, and an embedding added to it is counted once per rank.""" import unittest from types import SimpleNamespace -from unittest.mock import patch import torch from torch import nn @@ -26,6 +25,10 @@ def all_reduce(hidden_states): return hidden_states * TP_SIZE +# The group a deferred FFN output owes its sum over. +GROUP = SimpleNamespace(all_reduce=all_reduce) + + class DeferringLayer(nn.Module): """One TP rank's view of a decoder layer whose FFN output sums to one. It completes a reduction left by the previous layer, and leaves its own to the @@ -45,7 +48,10 @@ class DeferringLayer(nn.Module): residual = hidden_states if residual is None else hidden_states + residual if self.is_last_layer: return torch.ones_like(residual), residual - return UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE)), residual + return ( + UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE), group=GROUP), + residual, + ) class SumNorm(nn.Module): @@ -99,12 +105,6 @@ MODELS = (qwen3_vl_moe, qwen3_5) class TestDeepstackOnDeferredReduction(CustomTestCase): def setUp(self): - patcher = patch( - "sglang.srt.layers.communicator.deferred_post_experts_all_reduce", - all_reduce, - ) - patcher.start() - self.addCleanup(patcher.stop) self.embeds = torch.zeros(TOKENS, HIDDEN) def run_model(self, build, deepstack):