[Refactor] Step-3.5: complete the dense MLP's sum through ffn_exit (#41437)

This commit is contained in:
Cheng Wan
2026-09-27 16:53:57 -07:00
committed by GitHub
parent 0971450c16
commit 55d2ee5546
3 changed files with 31 additions and 17 deletions
+5 -11
View File
@@ -572,6 +572,9 @@ class Step3p5DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
# The dense MLP all-reduces its own output unless postprocess
# reduce-scatters it; it never leaves the sum to the next layer.
allow_deferred_ffn_reduction=self.use_moe,
)
self.layer_id = layer_id
@@ -618,18 +621,9 @@ class Step3p5DecoderLayer(nn.Module):
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
return ffn_exit.finish(hidden_states, residual)
# The dense MLP all-reduces its own output unless postprocess
# reduce-scatters it; it never leaves the sum to the next layer.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(
fuse_mlp_allreduce=False, mlp_reduce_scatter=mlp_reduce_scatter
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
return ffn_exit.finish(hidden_states, residual)
class Step3p5Model(nn.Module):
@@ -207,6 +207,17 @@ class TestLastLayerCommunicator(CustomTestCase):
layer_id == NUM_LAYERS - 1,
)
def test_step3p5_dense_layers_never_defer_their_sum(self):
"""A Step-3.5 dense layer's MLP all-reduces its own output unless
postprocess reduce-scatters it; its communicator is told it never leaves
the sum to the next layer. The MoE layers may."""
for layer_id in range(NUM_LAYERS):
with self.subTest(layer_id=layer_id):
passed, planned, _ = build("step3p5", NUM_LAYERS, layer_id)
self.assertIs(
passed["allow_deferred_ffn_reduction"], planned["is_layer_sparse"]
)
def test_draft_model_layer_is_last(self):
"""The single decoder layer of a NextN / MTP draft model is marked last,
whatever layer id and layer count it is built with."""
@@ -1,5 +1,6 @@
"""A Step-3.5 dense layer must leave its all-reduce out when postprocess
reduce-scatters its output; otherwise the output is summed twice."""
reduce-scatters its output; otherwise the output is summed twice. The layer's
FFN exit publishes the decision while the MLP runs, then completes it."""
import importlib
import unittest
@@ -8,6 +9,7 @@ from unittest.mock import MagicMock
import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.runtime_context import get_forward
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -27,24 +29,31 @@ def dense_layer(*, reduce_scatter):
seen["fuse_mlp_allreduce"] = get_forward().fuse_mlp_allreduce
return hidden_states
complete = MagicMock(side_effect=lambda h, r: (h, r))
communicator = SimpleNamespace(
prepare_attn=lambda h, r, fb, **_: (h, h if r is None else r),
prepare_mlp=lambda h, r, fb: (h, r),
should_use_reduce_scatter=lambda fb: reduce_scatter,
postprocess_layer=MagicMock(side_effect=lambda h, r, fb: (h, r)),
# A dense layer never leaves its sum to the next layer.
_select_ffn_completion=lambda fb: comm.FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
mlp_reduce_scatter=reduce_scatter,
complete=complete,
),
)
communicator.ffn_exit = lambda fb: comm.FfnExit(communicator, fb)
object.__setattr__(layer, "layer_communicator", communicator)
object.__setattr__(layer, "use_moe", False)
object.__setattr__(layer, "mlp", mlp)
object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"])
return layer, seen
return layer, seen, complete
class TestStep3p5DenseReduceScatter(CustomTestCase):
def test_dense_mlp_sees_the_postprocess_reduce_scatter(self):
for reduce_scatter in (False, True):
with self.subTest(reduce_scatter=reduce_scatter):
layer, seen = dense_layer(reduce_scatter=reduce_scatter)
layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter)
layer.forward(
positions=None,
hidden_states=torch.ones(2, 4),
@@ -53,7 +62,7 @@ class TestStep3p5DenseReduceScatter(CustomTestCase):
)
self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter)
self.assertFalse(seen["fuse_mlp_allreduce"])
layer.layer_communicator.postprocess_layer.assert_called_once()
complete.assert_called_once()
if __name__ == "__main__":