[Refactor] Step-3.5: complete the dense MLP's sum through ffn_exit (#41437)
This commit is contained in:
@@ -572,6 +572,9 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.post_attention_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
# The dense MLP all-reduces its own output unless postprocess
|
||||
# reduce-scatters it; it never leaves the sum to the next layer.
|
||||
allow_deferred_ffn_reduction=self.use_moe,
|
||||
)
|
||||
|
||||
self.layer_id = layer_id
|
||||
@@ -618,18 +621,9 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
# The dense MLP all-reduces its own output unless postprocess
|
||||
# reduce-scatters it; it never leaves the sum to the next layer.
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=False, mlp_reduce_scatter=mlp_reduce_scatter
|
||||
):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
|
||||
class Step3p5Model(nn.Module):
|
||||
|
||||
@@ -207,6 +207,17 @@ class TestLastLayerCommunicator(CustomTestCase):
|
||||
layer_id == NUM_LAYERS - 1,
|
||||
)
|
||||
|
||||
def test_step3p5_dense_layers_never_defer_their_sum(self):
|
||||
"""A Step-3.5 dense layer's MLP all-reduces its own output unless
|
||||
postprocess reduce-scatters it; its communicator is told it never leaves
|
||||
the sum to the next layer. The MoE layers may."""
|
||||
for layer_id in range(NUM_LAYERS):
|
||||
with self.subTest(layer_id=layer_id):
|
||||
passed, planned, _ = build("step3p5", NUM_LAYERS, layer_id)
|
||||
self.assertIs(
|
||||
passed["allow_deferred_ffn_reduction"], planned["is_layer_sparse"]
|
||||
)
|
||||
|
||||
def test_draft_model_layer_is_last(self):
|
||||
"""The single decoder layer of a NextN / MTP draft model is marked last,
|
||||
whatever layer id and layer count it is built with."""
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""A Step-3.5 dense layer must leave its all-reduce out when postprocess
|
||||
reduce-scatters its output; otherwise the output is summed twice."""
|
||||
reduce-scatters its output; otherwise the output is summed twice. The layer's
|
||||
FFN exit publishes the decision while the MLP runs, then completes it."""
|
||||
|
||||
import importlib
|
||||
import unittest
|
||||
@@ -8,6 +9,7 @@ from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.runtime_context import get_forward
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -27,24 +29,31 @@ def dense_layer(*, reduce_scatter):
|
||||
seen["fuse_mlp_allreduce"] = get_forward().fuse_mlp_allreduce
|
||||
return hidden_states
|
||||
|
||||
complete = MagicMock(side_effect=lambda h, r: (h, r))
|
||||
communicator = SimpleNamespace(
|
||||
prepare_attn=lambda h, r, fb, **_: (h, h if r is None else r),
|
||||
prepare_mlp=lambda h, r, fb: (h, r),
|
||||
should_use_reduce_scatter=lambda fb: reduce_scatter,
|
||||
postprocess_layer=MagicMock(side_effect=lambda h, r, fb: (h, r)),
|
||||
# A dense layer never leaves its sum to the next layer.
|
||||
_select_ffn_completion=lambda fb: comm.FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=reduce_scatter,
|
||||
complete=complete,
|
||||
),
|
||||
)
|
||||
communicator.ffn_exit = lambda fb: comm.FfnExit(communicator, fb)
|
||||
object.__setattr__(layer, "layer_communicator", communicator)
|
||||
object.__setattr__(layer, "use_moe", False)
|
||||
object.__setattr__(layer, "mlp", mlp)
|
||||
object.__setattr__(layer, "self_attn", lambda **kwargs: kwargs["hidden_states"])
|
||||
return layer, seen
|
||||
return layer, seen, complete
|
||||
|
||||
|
||||
class TestStep3p5DenseReduceScatter(CustomTestCase):
|
||||
def test_dense_mlp_sees_the_postprocess_reduce_scatter(self):
|
||||
for reduce_scatter in (False, True):
|
||||
with self.subTest(reduce_scatter=reduce_scatter):
|
||||
layer, seen = dense_layer(reduce_scatter=reduce_scatter)
|
||||
layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter)
|
||||
layer.forward(
|
||||
positions=None,
|
||||
hidden_states=torch.ones(2, 4),
|
||||
@@ -53,7 +62,7 @@ class TestStep3p5DenseReduceScatter(CustomTestCase):
|
||||
)
|
||||
self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter)
|
||||
self.assertFalse(seen["fuse_mlp_allreduce"])
|
||||
layer.layer_communicator.postprocess_layer.assert_called_once()
|
||||
complete.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user