diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 76ac7b44fe..2e02e6bb72 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -925,6 +925,11 @@ class LayerCommunicator: allow_reduce_scatter=self.allow_reduce_scatter, ) + def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": + """Decide once how this layer's FFN output reduction completes. Use the + result as a context manager around the FFN call, then call ``finish``.""" + return FfnExit(self, forward_batch) + def should_use_reduce_scatter(self, forward_batch: ForwardBatch): if not self.allow_reduce_scatter: return False @@ -1039,6 +1044,49 @@ def scatter_mode_layouts( } +class FfnExit: + """One FFN's reduction decision. Inside the ``with`` block it is published as + ``fuse_mlp_allreduce`` / ``mlp_reduce_scatter`` on ``get_forward()``.""" + + __slots__ = ( + "communicator", + "forward_batch", + "fuse_mlp_allreduce", + "mlp_reduce_scatter", + "_scope", + ) + + def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): + self.communicator = communicator + self.forward_batch = forward_batch + self.fuse_mlp_allreduce = ( + communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) + ) + self.mlp_reduce_scatter = communicator.should_use_reduce_scatter(forward_batch) + self._scope = get_forward().scoped( + fuse_mlp_allreduce=self.fuse_mlp_allreduce, + mlp_reduce_scatter=self.mlp_reduce_scatter, + ) + + def __enter__(self) -> "FfnExit": + self._scope.__enter__() + return self + + def __exit__(self, *exc_info): + return self._scope.__exit__(*exc_info) + + def finish( + self, hidden_states: torch.Tensor, residual: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Leave the reduction to the next layer's input norm, or postprocess.""" + if self.fuse_mlp_allreduce: + hidden_states._sglang_needs_allreduce_fusion = True + return hidden_states, residual + return self.communicator.postprocess_layer( + hidden_states, residual, self.forward_batch + ) + + @dataclass class CommunicateContext: process_group_sizes: Dict[ScatterMode, int] diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 6c30cd3908..e574d3a9af 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2692,17 +2692,6 @@ class DeepseekV2DecoderLayer(nn.Module): hidden_states, residual, forward_batch ) - fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( - forward_batch - ) - ) - - # For DP with padding, reduce scatter can be used instead of all-reduce. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - if isinstance(self.mlp, DeepseekV2MLP): gemm_output_zero_allocator = None @@ -2717,24 +2706,13 @@ class DeepseekV2DecoderLayer(nn.Module): else: _mlp_ctx = nullcontext() - with get_forward().scoped( - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, - ): - with _mlp_ctx: - hidden_states = self.mlp( - hidden_states, - forward_batch, - gemm_output_zero_allocator, - ) - - if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True - - if not fuse_mlp_allreduce: - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit, _mlp_ctx: + hidden_states = self.mlp( + hidden_states, + forward_batch, + gemm_output_zero_allocator, ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual, topk_indices diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 59adfa0d8f..e131432a36 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -83,7 +83,7 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_nextn import DeepseekV3ForCausalLMNextN from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM from sglang.srt.models.utils import WeightsMapper, apply_qk_norm -from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream +from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import ( add_prefix, cpu_has_amx_support, @@ -964,29 +964,9 @@ class Glm4MoeDecoderLayer(nn.Module): hidden_states, residual, forward_batch ) - fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( - forward_batch - ) - ) - - # For DP with padding, reduce scatter can be used instead of all-reduce. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - - with get_forward().scoped( - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, - ): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - - if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True - else: - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 6cdc9634c4..f443bf7fb0 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -63,7 +63,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream +from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import ( LazyValue, add_prefix, @@ -907,29 +907,9 @@ class Qwen3MoeDecoderLayer(nn.Module): hidden_states, residual, forward_batch ) - fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( - forward_batch - ) - ) - - # For DP with padding, reduce scatter can be used instead of all-reduce. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - - with get_forward().scoped( - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, - ): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states, forward_batch) - - if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True - else: - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + hidden_states, residual = ffn_exit.finish(hidden_states, residual) return hidden_states, residual diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py new file mode 100644 index 0000000000..e91a47dc07 --- /dev/null +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -0,0 +1,130 @@ +import unittest +from unittest.mock import MagicMock + +import torch + +from sglang.srt.layers.communicator import LayerCommunicator +from sglang.srt.runtime_context import get_forward +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def make_communicator(*, fuse, reduce_scatter, cls=LayerCommunicator): + """A communicator whose decisions and postprocess are stubbed, built + without the process-wide parallel state.""" + communicator = cls.__new__(cls) + if cls is LayerCommunicator: + communicator.should_fuse_mlp_allreduce_with_next_layer = MagicMock( + return_value=fuse + ) + communicator.should_use_reduce_scatter = MagicMock(return_value=reduce_scatter) + communicator.postprocess_layer = MagicMock( + side_effect=lambda hidden_states, residual, forward_batch: ( + hidden_states + 1, + residual, + ) + ) + return communicator + + +def published_flags(): + forward = get_forward() + return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter + + +class TestFfnExit(CustomTestCase): + def setUp(self): + self.hidden_states = torch.ones(3, 4) + self.residual = torch.zeros(3, 4) + self.forward_batch = object() + + def run_exit(self, communicator): + with communicator.ffn_exit(self.forward_batch) as ffn_exit: + seen = published_flags() + hidden_states = self.hidden_states * 2 + return seen, ffn_exit.finish(hidden_states, self.residual) + + def test_reduction_left_to_next_layer_is_marked(self): + communicator = make_communicator(fuse=True, reduce_scatter=False) + seen, (hidden_states, residual) = self.run_exit(communicator) + self.assertEqual(seen, (True, False)) + self.assertTrue(hidden_states._sglang_needs_allreduce_fusion) + self.assertIs(residual, self.residual) + communicator.postprocess_layer.assert_not_called() + + def test_postprocess_completes_other_exits(self): + for reduce_scatter in (False, True): + with self.subTest(reduce_scatter=reduce_scatter): + communicator = make_communicator( + fuse=False, reduce_scatter=reduce_scatter + ) + seen, (hidden_states, residual) = self.run_exit(communicator) + self.assertEqual(seen, (False, reduce_scatter)) + self.assertFalse( + getattr(hidden_states, "_sglang_needs_allreduce_fusion", False) + ) + communicator.postprocess_layer.assert_called_once() + torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1) + self.assertIs(residual, self.residual) + + def test_finish_applies_the_decision_the_ffn_saw(self): + """finish() applies the decision published to the FFN, even when the + decision methods would answer differently by then.""" + for fuse in (True, False): + with self.subTest(fuse=fuse): + communicator = make_communicator(fuse=fuse, reduce_scatter=False) + with communicator.ffn_exit(self.forward_batch) as ffn_exit: + seen = published_flags() + decide = communicator.should_fuse_mlp_allreduce_with_next_layer + decide.return_value = not fuse + hidden_states = self.hidden_states * 2 + hidden_states, _ = ffn_exit.finish(hidden_states, self.residual) + self.assertEqual(seen, (fuse, False)) + self.assertEqual( + getattr(hidden_states, "_sglang_needs_allreduce_fusion", False), + fuse, + ) + self.assertEqual(communicator.postprocess_layer.called, not fuse) + + def test_flags_are_restored_after_the_ffn(self): + before = published_flags() + self.run_exit(make_communicator(fuse=True, reduce_scatter=True)) + self.assertEqual(published_flags(), before) + + def test_subclass_decisions_are_used(self): + class NeverFuses(LayerCommunicator): + def should_fuse_mlp_allreduce_with_next_layer(self, forward_batch): + return False + + communicator = make_communicator( + fuse=True, reduce_scatter=False, cls=NeverFuses + ) + seen, _ = self.run_exit(communicator) + self.assertEqual(seen, (False, False)) + communicator.postprocess_layer.assert_called_once() + + def test_compiles_without_graph_breaks(self): + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator.should_fuse_mlp_allreduce_with_next_layer = lambda forward_batch: ( + True + ) + communicator.should_use_reduce_scatter = lambda forward_batch: False + forward_batch = object() + + def layer(hidden_states, residual): + with communicator.ffn_exit(forward_batch) as ffn_exit: + if get_forward().fuse_mlp_allreduce: + hidden_states = hidden_states * 2 + return ffn_exit.finish(hidden_states, residual) + + torch._dynamo.reset() + compiled = torch.compile(layer, backend="eager", fullgraph=True) + hidden_states, residual = compiled(self.hidden_states, self.residual) + torch.testing.assert_close(hidden_states, self.hidden_states * 2) + self.assertIs(residual, self.residual) + + +if __name__ == "__main__": + unittest.main()