From 7c9f74c6e1097903a2259152e26ed90053fbd7fd Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:15:58 -0700 Subject: [PATCH] [Fix] Step-3.5: stop dense layers from summing their output twice under DP attention (#41082) --- python/sglang/srt/models/step3p5.py | 10 +-- .../test_step3p5_dense_reduce_scatter.py | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 test/registered/unit/models/test_step3p5_dense_reduce_scatter.py diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 604cbfe1dc..aff11d8176 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -633,10 +633,12 @@ class Step3p5DecoderLayer(nn.Module): if not fuse_mlp_allreduce and not mlp_reduce_scatter: hidden_states = tensor_model_parallel_all_reduce(hidden_states) else: - hidden_states = self.mlp(hidden_states) - # Dense MLP uses reduce_results=True, so the output is already - # all-reduced. Do NOT set the fusion flag — otherwise the next - # layer would all-reduce again, multiplying values by world_size. + # The dense MLP all-reduces its own output unless postprocess + # reduce-scatters it; it never leaves the sum to the next layer. + with get_forward().scoped( + fuse_mlp_allreduce=False, mlp_reduce_scatter=mlp_reduce_scatter + ): + hidden_states = self.mlp(hidden_states) fuse_mlp_allreduce = False if fuse_mlp_allreduce: diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py new file mode 100644 index 0000000000..4e46802d36 --- /dev/null +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -0,0 +1,61 @@ +"""A Step-3.5 dense layer must leave its all-reduce out when postprocess +reduce-scatters its output; otherwise the output is summed twice.""" + +import importlib +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock + +import torch + +from sglang.srt.runtime_context import get_forward +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def dense_layer(*, reduce_scatter): + step3p5 = importlib.import_module("sglang.srt.models.step3p5") + layer = step3p5.Step3p5DecoderLayer.__new__(step3p5.Step3p5DecoderLayer) + torch.nn.Module.__init__(layer) + seen = {} + + def mlp(hidden_states): + # RowParallelLinear skips its all-reduce when either flag is published. + seen["mlp_reduce_scatter"] = get_forward().mlp_reduce_scatter + seen["fuse_mlp_allreduce"] = get_forward().fuse_mlp_allreduce + return hidden_states + + communicator = SimpleNamespace( + prepare_attn=lambda h, r, fb, **_: (h, h if r is None else r), + prepare_mlp=lambda h, r, fb: (h, r), + should_fuse_mlp_allreduce_with_next_layer=lambda fb: False, + should_use_reduce_scatter=lambda fb: reduce_scatter, + postprocess_layer=MagicMock(side_effect=lambda h, r, fb: (h, r)), + ) + object.__setattr__(layer, "layer_communicator", communicator) + object.__setattr__(layer, "use_moe", False) + object.__setattr__(layer, "mlp", mlp) + object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"]) + return layer, seen + + +class TestStep3p5DenseReduceScatter(CustomTestCase): + def test_dense_mlp_sees_the_postprocess_reduce_scatter(self): + for reduce_scatter in (False, True): + with self.subTest(reduce_scatter=reduce_scatter): + layer, seen = dense_layer(reduce_scatter=reduce_scatter) + layer.forward( + positions=None, + hidden_states=torch.ones(2, 4), + forward_batch=None, + residual=None, + ) + self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter) + self.assertFalse(seen["fuse_mlp_allreduce"]) + layer.layer_communicator.postprocess_layer.assert_called_once() + + +if __name__ == "__main__": + unittest.main()