From 55d2ee554635b8ac5db58b30ae828a83bd0cf27b Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:53:57 -0700 Subject: [PATCH] [Refactor] Step-3.5: complete the dense MLP's sum through ffn_exit (#41437) --- python/sglang/srt/models/step3p5.py | 16 +++++--------- .../models/test_last_layer_communicator.py | 11 ++++++++++ .../test_step3p5_dense_reduce_scatter.py | 21 +++++++++++++------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 35dcf19878..83e2b23b06 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -572,6 +572,9 @@ class Step3p5DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + # The dense MLP all-reduces its own output unless postprocess + # reduce-scatters it; it never leaves the sum to the next layer. + allow_deferred_ffn_reduction=self.use_moe, ) self.layer_id = layer_id @@ -618,18 +621,9 @@ class Step3p5DecoderLayer(nn.Module): hidden_states = tensor_model_parallel_all_reduce(hidden_states) return ffn_exit.finish(hidden_states, residual) - # The dense MLP all-reduces its own output unless postprocess - # reduce-scatters it; it never leaves the sum to the next layer. - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - with get_forward().scoped( - fuse_mlp_allreduce=False, mlp_reduce_scatter=mlp_reduce_scatter - ): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.mlp(hidden_states) - return self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) + return ffn_exit.finish(hidden_states, residual) class Step3p5Model(nn.Module): diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index 2bf4d313b2..89e6cdd117 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -207,6 +207,17 @@ class TestLastLayerCommunicator(CustomTestCase): layer_id == NUM_LAYERS - 1, ) + def test_step3p5_dense_layers_never_defer_their_sum(self): + """A Step-3.5 dense layer's MLP all-reduces its own output unless + postprocess reduce-scatters it; its communicator is told it never leaves + the sum to the next layer. The MoE layers may.""" + for layer_id in range(NUM_LAYERS): + with self.subTest(layer_id=layer_id): + passed, planned, _ = build("step3p5", NUM_LAYERS, layer_id) + self.assertIs( + passed["allow_deferred_ffn_reduction"], planned["is_layer_sparse"] + ) + def test_draft_model_layer_is_last(self): """The single decoder layer of a NextN / MTP draft model is marked last, whatever layer id and layer count it is built with.""" diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py index 97428481a1..2fcc3cb5ea 100644 --- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -1,5 +1,6 @@ """A Step-3.5 dense layer must leave its all-reduce out when postprocess -reduce-scatters its output; otherwise the output is summed twice.""" +reduce-scatters its output; otherwise the output is summed twice. The layer's +FFN exit publishes the decision while the MLP runs, then completes it.""" import importlib import unittest @@ -8,6 +9,7 @@ from unittest.mock import MagicMock import torch +from sglang.srt.layers import communicator as comm from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -27,24 +29,31 @@ def dense_layer(*, reduce_scatter): seen["fuse_mlp_allreduce"] = get_forward().fuse_mlp_allreduce return hidden_states + complete = MagicMock(side_effect=lambda h, r: (h, r)) communicator = SimpleNamespace( prepare_attn=lambda h, r, fb, **_: (h, h if r is None else r), prepare_mlp=lambda h, r, fb: (h, r), - should_use_reduce_scatter=lambda fb: reduce_scatter, - postprocess_layer=MagicMock(side_effect=lambda h, r, fb: (h, r)), + # A dense layer never leaves its sum to the next layer. + _select_ffn_completion=lambda fb: comm.FfnCompletion( + defer_moe_finalize=False, + fuse_mlp_allreduce=False, + mlp_reduce_scatter=reduce_scatter, + complete=complete, + ), ) + communicator.ffn_exit = lambda fb: comm.FfnExit(communicator, fb) object.__setattr__(layer, "layer_communicator", communicator) object.__setattr__(layer, "use_moe", False) object.__setattr__(layer, "mlp", mlp) object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"]) - return layer, seen + return layer, seen, complete class TestStep3p5DenseReduceScatter(CustomTestCase): def test_dense_mlp_sees_the_postprocess_reduce_scatter(self): for reduce_scatter in (False, True): with self.subTest(reduce_scatter=reduce_scatter): - layer, seen = dense_layer(reduce_scatter=reduce_scatter) + layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter) layer.forward( positions=None, hidden_states=torch.ones(2, 4), @@ -53,7 +62,7 @@ class TestStep3p5DenseReduceScatter(CustomTestCase): ) self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter) self.assertFalse(seen["fuse_mlp_allreduce"]) - layer.layer_communicator.postprocess_layer.assert_called_once() + complete.assert_called_once() if __name__ == "__main__":