From e31cbc01a2932d50cd16ad601ee1001b98970f38 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:41:14 -0700 Subject: [PATCH] [Fix] Stop deferring the last layer's FFN all-reduce in five models (#40868) --- python/sglang/srt/models/bailing_moe.py | 5 +- .../sglang/srt/models/bailing_moe_linear.py | 1 + python/sglang/srt/models/bailing_moe_nextn.py | 1 + python/sglang/srt/models/minimax_m3.py | 1 + python/sglang/srt/models/qwen3_next.py | 2 + python/sglang/srt/models/step3p5.py | 3 +- python/sglang/srt/models/step3p5_mtp.py | 5 +- .../models/test_last_layer_communicator.py | 203 ++++++++++++++++++ 8 files changed, 217 insertions(+), 4 deletions(-) create mode 100644 test/registered/unit/models/test_last_layer_communicator.py diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 06603253b1..a61cf61b2b 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -644,6 +644,7 @@ class BailingMoEBlock(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, + is_nextn: bool = False, ): super().__init__() self.config = config @@ -681,7 +682,7 @@ class BailingMoEBlock(nn.Module): is_next_layer_sparse=is_next_layer_sparse, ) - self.is_last_layer = self.layer_id == config.num_hidden_layers - 1 + self.is_last_layer = is_nextn or (self.layer_id == config.num_hidden_layers - 1) if self.is_layer_sparse: self.mlp = BailingMoESparseMoeBlock( @@ -712,7 +713,7 @@ class BailingMoEBlock(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(self.layer_id == self.config.num_hidden_layers - 1), + is_last_layer=self.is_last_layer, ) def _is_layer_sparse( diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 1777dc3971..03e6072815 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -835,6 +835,7 @@ class BailingMoELinearDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=False, + is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1), qkv_latent_func=qkv_latent_func, ) diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index 9e9eecb9ac..8abb0a1bb6 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -127,6 +127,7 @@ class BailingMoEModelNextN(nn.Module): 0, quant_config=quant_config, prefix=add_prefix("decoder", prefix), + is_nextn=True, ) self.shared_head = nn.Module() diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 7a6f2ea6d5..bc4b8dc97a 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1361,6 +1361,7 @@ class MiniMaxM3DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 4ba1798748..e0e9b22662 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -561,6 +561,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + is_last_layer=(layer_id == config.num_hidden_layers - 1), ) def forward( @@ -733,6 +734,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, + is_last_layer=(layer_id == config.num_hidden_layers - 1), ) self.alt_stream = alt_stream diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index d9f68f3633..2c6cfb6dfb 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -464,6 +464,7 @@ class Step3p5DecoderLayer(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, + is_nextn: bool = False, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -575,7 +576,7 @@ class Step3p5DecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.post_attention_layernorm, allow_reduce_scatter=True, - is_last_layer=(layer_id == config.num_hidden_layers - 1), + is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1), ) self.layer_id = layer_id diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index d3d059e1ef..491eea15d2 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -80,7 +80,10 @@ class Step3p5AMultiTokenPredictor(nn.Module): self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) self.shared_head = SharedHead(config=config, quant_config=quant_config) self.mtp_block = Step3p5DecoderLayer( - config=config, layer_id=layer_id, prefix=f"{prefix}.mtp_block" + config=config, + layer_id=layer_id, + prefix=f"{prefix}.mtp_block", + is_nextn=True, ) self.lm_head = self.shared_head.head diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py new file mode 100644 index 0000000000..f88af802a1 --- /dev/null +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -0,0 +1,203 @@ +import importlib +import sys +import types +import unittest +from types import ModuleType +from unittest.mock import MagicMock, patch + +import torch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +NUM_LAYERS = 4 + + +def import_model(name): + if importlib.util.find_spec("vllm") is not None: + return importlib.import_module(f"sglang.srt.models.{name}") + # CPU CI omits vLLM; the Bailing modules import an AWQ kernel from it. + vllm = ModuleType("vllm") + vllm.__path__ = [] + custom_ops = ModuleType("vllm._custom_ops") + custom_ops.awq_dequantize = MagicMock() + with patch.dict(sys.modules, {"vllm": vllm, "vllm._custom_ops": custom_ops}): + return importlib.import_module(f"sglang.srt.models.{name}") + + +class StubModule(torch.nn.Module): + def __getattr__(self, name): + try: + return super().__getattr__(name) + except AttributeError: + return MagicMock() + + +def stub(*args, **kwargs): + return StubModule() + + +PARALLEL = types.SimpleNamespace( + tp_size=1, + tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + attn_dp_size=1, + attn_dp_rank=0, + moe_ep_size=1, + moe_tp_size=1, + moe_dp_size=1, +) + + +def qwen3_next_config(num_layers): + from sglang.srt.configs.qwen3_next import Qwen3NextConfig + + return Qwen3NextConfig(num_hidden_layers=num_layers) + + +def bailing_hybrid_config(num_layers): + from sglang.srt.configs.bailing_hybrid import BailingHybridConfig + + return BailingHybridConfig(num_hidden_layers=num_layers, attention_type=1) + + +def bailing_moe_config(num_layers): + from sglang.srt.configs.bailing_moe_v2 import BailingMoeV2Config + + return BailingMoeV2Config(num_hidden_layers=num_layers) + + +def step3p5_config(num_layers): + from sglang.srt.configs.step3p5 import Step3p5Config + + # Per-layer lists also cover the MTP layer, whose id is num_layers. + return Step3p5Config( + num_hidden_layers=num_layers, + layer_types=["full_attention"] * (num_layers + 1), + moe_layers_enum=",".join(str(i) for i in range(1, num_layers + 1)), + swiglu_limits_shared=None, + rope_theta=[10000.0] * (num_layers + 1), + partial_rotary_factors=[1.0] * (num_layers + 1), + use_head_wise_attn_gate=False, + share_expert_dim=16, + ) + + +def minimax_m3_config(num_layers): + from transformers import PretrainedConfig + + return PretrainedConfig( + num_hidden_layers=num_layers, + hidden_size=16, + rms_norm_eps=1e-6, + dense_intermediate_size=32, + ) + + +# name: (module, decoder layer class, config factory, submodules to stub, +# (config layer count, layer id) of the draft model's layer or None) +CASES = { + "qwen3_next_linear": ( + "qwen3_next", + "Qwen3HybridLinearDecoderLayer", + qwen3_next_config, + ["Qwen3GatedDeltaNet", "Qwen2MoeSparseMoeBlock", "Qwen2MoeMLP"], + (1, 0), + ), + "qwen3_next_attention": ( + "qwen3_next", + "Qwen3HybridAttentionDecoderLayer", + qwen3_next_config, + [ + "Qwen2MoeSparseMoeBlock", + "Qwen2MoeMLP", + "QKVParallelLinear", + "RowParallelLinear", + "RadixAttention", + "get_rope", + ], + (1, 0), + ), + "bailing_moe_linear": ( + "bailing_moe_linear", + "BailingMoELinearDecoderLayer", + bailing_hybrid_config, + [ + "BailingMLP", + "BailingMoE", + "BailingMoEAttention", + "BailingMoELinearAttention", + "DsV3MLA", + ], + (NUM_LAYERS, 0), + ), + "bailing_moe": ( + "bailing_moe", + "BailingMoEBlock", + bailing_moe_config, + ["BailingMoEAttention", "BailingMoEMLP", "BailingMoESparseMoeBlock"], + (NUM_LAYERS, 0), + ), + "step3p5": ( + "step3p5", + "Step3p5DecoderLayer", + step3p5_config, + ["Step3p5Attention", "Step3p5MLP", "Step3p5MoEMLP"], + (NUM_LAYERS, NUM_LAYERS), + ), + "minimax_m3": ( + "minimax_m3", + "MiniMaxM3DecoderLayer", + minimax_m3_config, + ["MiniMaxM3Attention", "MiniMaxM3MLP", "MiniMaxM3MoE"], + None, + ), +} + + +def is_last_layer_passed(case, num_layers, layer_id, **kwargs): + module_name, class_name, make_config, stubs, _ = CASES[case] + module = import_model(module_name) + communicator = MagicMock() + patches = dict(LayerCommunicator=communicator, LayerScatterModes=MagicMock()) + patches.update({name: stub for name in stubs}) + if hasattr(module, "get_parallel"): + patches["get_parallel"] = lambda: PARALLEL + with patch.multiple(module, **patches): + getattr(module, class_name)( + make_config(num_layers), layer_id=layer_id, **kwargs + ) + communicator.assert_called_once() + return communicator.call_args.kwargs.get("is_last_layer", False) + + +class TestLastLayerCommunicator(CustomTestCase): + def test_only_the_last_layer_is_last(self): + """With all-reduce fusion on, only the model's last layer is marked last, + so its FFN all-reduce is never left to a next layer that does not exist.""" + for case in CASES: + for layer_id in range(NUM_LAYERS): + with self.subTest(case=case, layer_id=layer_id): + self.assertEqual( + is_last_layer_passed(case, NUM_LAYERS, layer_id), + layer_id == NUM_LAYERS - 1, + ) + + def test_draft_model_layer_is_last(self): + """The single decoder layer of a NextN / MTP draft model is marked last, + whatever layer id and layer count it is built with.""" + for case, (*_, draft) in CASES.items(): + if draft is None: + continue + num_layers, layer_id = draft + with self.subTest(case=case): + self.assertTrue( + is_last_layer_passed(case, num_layers, layer_id, is_nextn=True) + ) + + +if __name__ == "__main__": + unittest.main()