diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index c3e681bb86..b9a7f224f4 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -1127,17 +1127,37 @@ class LayerCommunicator: self._sp_variant is not None and get_forward().sp_active ) - def _reduce_scatter_step( + def _postprocess_dp_step( self, forward_batch: ForwardBatch ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: - """The reduce-scatter postprocess would run on this layer's FFN output; - None when it runs only a scatter, or anything else.""" + """The reduce-scatter that brings this layer's FFN output back to this + rank's tokens under attention DP; None when the base postprocess would + only scatter, or does not move tokens.""" + if not self._postprocess_scatters_to_local_tokens: + return None return _reduce_and_redistribute_output_step( forward_batch, allow_reduce_scatter=self.allow_reduce_scatter, is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) + def _ffn_leaves_sum_to_reduce_scatter( + self, forward_batch: ForwardBatch, dp_step: Optional[Callable] + ) -> bool: + """Whether the FFN leaves its sum out because a reduce-scatter completes + it: the attention-DP one ``dp_step`` names, or the CP / input-scattered + one.""" + if not self.allow_reduce_scatter: + return False + if dp_step is not None: + return True + # Prefill CP predicates must stay out of decode graph capture. + if forward_batch.forward_mode.is_context_parallel_extend() and ( + dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch) + ): + return True + return get_attn_tp_context().input_scattered and not self.is_last_layer + def ffn_reduction_group(self) -> GroupCoordinator: """The group this layer's FFN output owes its sum over: the MoE output's group on a sparse layer, the TP group a dense MLP reduces over.""" @@ -1147,47 +1167,76 @@ class LayerCommunicator: def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion": """Decide once, before the FFN runs, what it skips and what completes its - output: the next layer's input, or this layer's postprocess.""" + output: the next layer's input, or this layer's postprocess step.""" + dp_step = self._postprocess_dp_step(forward_batch) + mlp_reduce_scatter = self._ffn_leaves_sum_to_reduce_scatter( + forward_batch, dp_step + ) defer_moe_finalize = self.should_defer_moe_finalize(forward_batch) # Deferring implies fusing: a handoff skips the post-experts all-reduce. - fuse_mlp_allreduce = defer_moe_finalize or self.should_defer_ffn_reduction( - forward_batch + fuse_mlp_allreduce = defer_moe_finalize or self._ffn_sum_moves_to_next_layer( + forward_batch, mlp_reduce_scatter=mlp_reduce_scatter, dp_step=dp_step ) - mlp_reduce_scatter = self.should_use_reduce_scatter(forward_batch) if fuse_mlp_allreduce: group = self.ffn_reduction_group() if self._postprocess_scatters_to_local_tokens: # Under attention DP the next layer also brings the sum back to # this rank's tokens. - leave = partial( + wrap = partial( UnreducedOutput, reduce_and_redistribute=partial( _all_reduce_then_to_local_tokens, group, forward_batch ), ) else: - leave = partial(UnreducedOutput, group=group) - elif not self.is_last_layer and self._local_token_move_can_go_to_next_layer(): - step = self._reduce_scatter_step(forward_batch) - leave = ( - None - if step is None - else partial( + wrap = partial(UnreducedOutput, group=group) + complete = partial(_leave_to_next_layer, wrap) + elif ( + dp_step is not None + and not self.is_last_layer + and self._local_token_move_can_go_to_next_layer() + ): + complete = partial( + _leave_to_next_layer, + partial( UnreducedOutput, reduce_and_redistribute=partial( - _to_local_tokens, step, forward_batch + _to_local_tokens, dp_step, forward_batch ), - ) + ), ) else: - leave = None + complete = partial( + self._complete_ffn_output_now, + forward_batch=forward_batch, + dp_step=dp_step, + ) return FfnCompletion( defer_moe_finalize=defer_moe_finalize, fuse_mlp_allreduce=fuse_mlp_allreduce, mlp_reduce_scatter=mlp_reduce_scatter, - leave=leave, + complete=complete, ) + def _complete_ffn_output_now( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + *, + forward_batch: ForwardBatch, + dp_step: Optional[Callable], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """This layer's postprocess, run with the attention-DP step the FFN exit + chose.""" + if self._local_token_move_can_go_to_next_layer(): + return ( + _to_local_tokens( + dp_step or _redistribute_output, forward_batch, hidden_states + ), + residual, + ) + return self.postprocess_layer(hidden_states, residual, forward_batch) + def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": """Decide once how this layer's FFN output reduction completes. Use the result as a context manager around the FFN call, then call ``finish``.""" @@ -1205,27 +1254,11 @@ class LayerCommunicator: return reduce_output(hidden_states), residual def should_use_reduce_scatter(self, forward_batch: ForwardBatch): - if not self.allow_reduce_scatter: - return False - if ( - self._communicate_summable_tensor_pair_fn - is CommunicateSummableTensorPairFn._scatter_hidden_states - ): - if should_use_dp_reduce_scatterv(): - return True - if ( - forward_batch.dp_padding_mode.is_max_len() - and can_use_dp_reduce_scatter() - ): - return True - # Prefill CP predicates must stay out of decode graph capture. - if forward_batch.forward_mode.is_context_parallel_extend() and ( - dsa_use_prefill_cp(forward_batch) or is_mla_cp_active(forward_batch) - ): - return True - if get_attn_tp_context().input_scattered and not self.is_last_layer: - return True - return False + """Whether the FFN leaves its sum to a reduce-scatter, for layers that run + their FFN outside ffn_exit.""" + return self._ffn_leaves_sum_to_reduce_scatter( + forward_batch, self._postprocess_dp_step(forward_batch) + ) def should_defer_moe_finalize( self, forward_batch: ForwardBatch, m: int | None = None @@ -1293,7 +1326,13 @@ class LayerCommunicator: and (self._context.tp_size > 1) ) - def should_defer_ffn_reduction(self, forward_batch: ForwardBatch) -> bool: + def _ffn_sum_moves_to_next_layer( + self, + forward_batch: ForwardBatch, + *, + mlp_reduce_scatter: bool, + dp_step: Optional[Callable], + ) -> bool: """Whether the FFN leaves its output's all-reduce to the next layer's input norm: whenever the fused kernel takes it, and otherwise when the next layer would run the same all-reduce the FFN itself would have.""" @@ -1304,15 +1343,12 @@ class LayerCommunicator: and not self.is_last_layer and self._ffn_sum_can_move_to_next_layer() and _unfused_completion_matches_the_ffn(forward_batch) - and not self.should_use_reduce_scatter(forward_batch) + and not mlp_reduce_scatter # Under attention DP the next layer must also run postprocess's # scatter back to this rank's tokens, and nothing more. and ( not is_dp_attention_enabled() - or ( - self._local_token_move_can_go_to_next_layer() - and self._reduce_scatter_step(forward_batch) is None - ) + or (self._local_token_move_can_go_to_next_layer() and dp_step is None) ) ) @@ -1346,13 +1382,21 @@ def scatter_mode_layouts( class FfnCompletion(msgspec.Struct, frozen=True): """One FFN's reduction decision. The flags are published while the FFN runs; - ``leave`` wraps its output for the next layer's input to complete, or is None - when this layer's postprocess completes it.""" + ``complete(hidden_states, residual)`` then either wraps the output for the + next layer's input to complete, or runs this layer's postprocess step.""" defer_moe_finalize: bool fuse_mlp_allreduce: bool mlp_reduce_scatter: bool - leave: Optional[Callable[[torch.Tensor], UnreducedOutput]] = None + complete: Callable[[torch.Tensor, torch.Tensor], Tuple] + + +def _leave_to_next_layer( + wrap: Callable[[torch.Tensor], UnreducedOutput], + hidden_states: torch.Tensor, + residual: torch.Tensor, +) -> Tuple[UnreducedOutput, torch.Tensor]: + return wrap(hidden_states), residual class FfnExit: @@ -1366,7 +1410,7 @@ class FfnExit: "defer_moe_finalize", "fuse_mlp_allreduce", "mlp_reduce_scatter", - "_leave", + "_complete", "_scope", ) @@ -1377,7 +1421,7 @@ class FfnExit: self.defer_moe_finalize = completion.defer_moe_finalize self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce self.mlp_reduce_scatter = completion.mlp_reduce_scatter - self._leave = completion.leave + self._complete = completion.complete self._scope = get_forward().scoped( fuse_mlp_allreduce=self.fuse_mlp_allreduce, mlp_reduce_scatter=self.mlp_reduce_scatter, @@ -1399,11 +1443,7 @@ class FfnExit: # A deferred MoE finalize handoff, consumed by the next prepare_attn. assert self.defer_moe_finalize, "unrequested deferred MoE handoff" return hidden_states, residual - if self._leave is not None: - return self._leave(hidden_states), residual - return self.communicator.postprocess_layer( - hidden_states, residual, self.forward_batch - ) + return self._complete(hidden_states, residual) @dataclass @@ -2038,7 +2078,6 @@ class CommunicateSummableTensorPairFn: allow_reduce_scatter: bool = False, is_layer_sparse: bool = False, ): - local_hidden_states = get_local_dp_buffer(_dp_scatter_group()) step = ( _reduce_and_redistribute_output_step( forward_batch, @@ -2047,8 +2086,7 @@ class CommunicateSummableTensorPairFn: ) or _redistribute_output ) - step(local_hidden_states, hidden_states, forward_batch) - return local_hidden_states, residual + return _to_local_tokens(step, forward_batch, hidden_states), residual @staticmethod def _gather( diff --git a/python/sglang/srt/layers/communicator_mhc.py b/python/sglang/srt/layers/communicator_mhc.py index e8c3052b60..64e9c94634 100644 --- a/python/sglang/srt/layers/communicator_mhc.py +++ b/python/sglang/srt/layers/communicator_mhc.py @@ -29,6 +29,7 @@ from sglang.srt.layers.communicator import ( CommunicateContext, CommunicateSimpleFn, CommunicateSummableTensorPairFn, + FfnCompletion, LayerCommunicator, LayerScatterModes, MlpInputKind, @@ -513,11 +514,19 @@ class MHCLayerCommunicator(LayerCommunicator): return hidden_states, residual - def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): - return False - - def should_defer_ffn_reduction(self, forward_batch): - return False + def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion: + """An MHC layer's own postprocess completes its FFN output, combining the + hyper-connection streams; nothing is left to the next layer.""" + return FfnCompletion( + defer_moe_finalize=False, + fuse_mlp_allreduce=False, + mlp_reduce_scatter=self.should_use_reduce_scatter(forward_batch), + complete=partial( + self._complete_ffn_output_now, + forward_batch=forward_batch, + dp_step=None, + ), + ) def should_use_reduce_scatter(self, forward_batch: ForwardBatch): if not self.allow_reduce_scatter: diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 3fdfc56e6a..827307cdaa 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -38,12 +38,17 @@ def make_communicator( without the process-wide parallel state.""" communicator = cls.__new__(cls) if cls is LayerCommunicator: - communicator.should_defer_ffn_reduction = MagicMock(return_value=fuse) - communicator.should_use_reduce_scatter = MagicMock(return_value=reduce_scatter) + communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse) + else: + # A subclass decides through its own hooks; no TP left to defer over. + communicator._context = types.SimpleNamespace(tp_size=1) + communicator._ffn_leaves_sum_to_reduce_scatter = MagicMock( + return_value=reduce_scatter + ) communicator.is_last_layer = False communicator._sp_variant = None communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens - communicator._reduce_scatter_step = MagicMock(return_value=reduce_scatter_step) + communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step) communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) communicator.postprocess_layer = MagicMock( side_effect=lambda hidden_states, residual, forward_batch: ( @@ -116,7 +121,6 @@ class TestFfnExit(CustomTestCase): bound = hidden_states.reduce_and_redistribute self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) self.assertEqual(bound.args, (group, self.forward_batch)) - communicator._reduce_scatter_step.assert_not_called() communicator.postprocess_layer.assert_not_called() group.all_reduce.assert_not_called() @@ -141,7 +145,7 @@ class TestFfnExit(CustomTestCase): communicator = make_communicator(fuse=fuse, reduce_scatter=False) with communicator.ffn_exit(self.forward_batch) as ffn_exit: seen = published_flags() - decide = communicator.should_defer_ffn_reduction + decide = communicator._ffn_sum_moves_to_next_layer decide.return_value = not fuse hidden_states = self.hidden_states * 2 hidden_states, _ = ffn_exit.finish(hidden_states, self.residual) @@ -156,7 +160,7 @@ class TestFfnExit(CustomTestCase): def test_subclass_decisions_are_used(self): class NeverDefers(LayerCommunicator): - def should_defer_ffn_reduction(self, forward_batch): + def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): return False communicator = make_communicator( @@ -268,8 +272,11 @@ class TestSelectFfnCompletion(CustomTestCase): communicator._postprocess_scatters_to_local_tokens = scatters communicator.allow_reduce_scatter = True communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) - communicator.should_defer_ffn_reduction = lambda forward_batch: fuse - communicator.should_use_reduce_scatter = lambda forward_batch: not fuse + communicator._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: fuse + communicator._ffn_leaves_sum_to_reduce_scatter = lambda forward_batch, dp_step: ( + not fuse + ) + communicator._complete_ffn_output_now = lambda h, r, **_: ("now", r) self.group = make_group() communicator.ffn_reduction_group = lambda: self.group return communicator @@ -279,9 +286,8 @@ class TestSelectFfnCompletion(CustomTestCase): comm, "_reduce_and_redistribute_output_step", return_value=step ): completion = communicator._select_ffn_completion(forward_batch) - if completion.leave is None: - return None - return completion.leave(torch.ones(3, 4)) + hidden_states, _ = completion.complete(torch.ones(3, 4), None) + return None if hidden_states == "now" else hidden_states def test_a_reduce_scatter_is_bound_for_the_next_layer(self): forward_batch = object() diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 0d1ed19c96..02417a6580 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -322,7 +322,6 @@ class TestDeferFfnReduction(CustomTestCase): ): communicator = _fake_communicator() communicator.is_last_layer = is_last_layer - communicator.should_use_reduce_scatter = lambda forward_batch: reduce_scatter communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._sp_variant = object() if sp_active else None communicator.allow_reduce_scatter = True @@ -379,7 +378,9 @@ class TestDeferFfnReduction(CustomTestCase): moe_ep_size=1, moe_tp_size=4, moe_dp_size=1, tp_size=4 ), ): - return communicator.should_defer_ffn_reduction(forward_batch) + return communicator._ffn_sum_moves_to_next_layer( + forward_batch, mlp_reduce_scatter=reduce_scatter, dp_step=step + ) def test_defers_whether_or_not_the_fused_kernel_takes_the_batch(self): self.assertTrue(self._should_defer(fused=True)) diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 067c163af4..577d0780a4 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -49,11 +49,13 @@ class DeferringLayer(nn.Module): self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = ( lambda batch: defer ) - self.layer_communicator.should_defer_ffn_reduction = lambda batch: defer + self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer self.layer_communicator.is_last_layer = False self.layer_communicator._postprocess_scatters_to_local_tokens = False self.layer_communicator.ffn_reduction_group = lambda: GROUP - self.layer_communicator.should_use_reduce_scatter = lambda batch: False + self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = ( + lambda batch, dp_step: False + ) self.layer_communicator.postprocess_layer = lambda hidden, residual, batch: ( all_reduce(hidden), residual,