diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 1f8dff1b0f..ad5fa0e15b 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1283,9 +1283,12 @@ class ModelConfig: self.v_head_dim = tc.v_head_dim self.qk_nope_head_dim = tc.qk_nope_head_dim self._init_mla_scaling(getattr(tc, "rope_scaling", None)) - elif ( - "BailingMoeV2_5ForCausalLM" in self.hf_config.architectures - or "BailingMoeForCausalLMNextN" in self.hf_config.architectures + elif "BailingMoeV2_5ForCausalLM" in self.hf_config.architectures or ( + # Every Bailing draft is renamed to BailingMoeForCausalLMNextN; only + # the MLA ones (V2.5 and V3) carry kv_lora_rank, V2 keeps its GQA + # shapes. + "BailingMoeForCausalLMNextN" in self.hf_config.architectures + and getattr(self.hf_text_config, "kv_lora_rank", None) is not None ): self.head_dim = self.hf_text_config.head_dim self.attention_arch = AttentionArch.MLA diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 4c7c94f612..1c22f82353 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -660,7 +660,7 @@ class BailingMoEBlock(nn.Module): self.attn_tp_rank = get_parallel().attn_tp_rank self.is_layer_sparse = self._is_layer_sparse( - config, layer_id=layer_id, is_nextn=False + config, layer_id=layer_id, is_nextn=is_nextn ) is_previous_layer_sparse = self._is_layer_sparse( config, layer_id=layer_id - 1, is_nextn=False @@ -671,7 +671,8 @@ class BailingMoEBlock(nn.Module): self.layer_scatter_modes = LayerScatterModes.init_new( layer_id=layer_id, - num_layers=config.num_hidden_layers, + # A NextN draft is a one-layer model. + num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_layer_sparse, is_previous_layer_sparse=is_previous_layer_sparse, is_next_layer_sparse=is_next_layer_sparse, diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index c293384473..be2f3efd4a 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -781,7 +781,7 @@ class BailingMoELinearDecoderLayer(nn.Module): self.expert_num = config.num_experts self.hidden_size = config.hidden_size - is_moe_layer = self._is_layer_sparse(config, self.layer_id) + is_moe_layer = self._is_layer_sparse(config, self.layer_id, is_nextn=is_nextn) is_previous_moe_layer = self._is_layer_sparse(config, self.layer_id - 1) is_next_layer_moe_layer = self._is_layer_sparse(config, self.layer_id + 1) if self.expert_num == 1: @@ -815,7 +815,8 @@ class BailingMoELinearDecoderLayer(nn.Module): self.layer_scatter_modes = LayerScatterModes.init_new( layer_id=layer_id, - num_layers=config.num_hidden_layers, + # A NextN draft is a one-layer model. + num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=is_moe_layer, is_previous_layer_sparse=is_previous_moe_layer, is_next_layer_sparse=is_next_layer_moe_layer, diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index aff11d8176..ba4688a931 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -562,11 +562,10 @@ class Step3p5DecoderLayer(nn.Module): config.hidden_size, eps=config.rms_norm_eps ) + # An MTP draft is a one-layer model; layer_id still indexes its config. self.layer_scatter_modes = LayerScatterModes.init_new( - layer_id=layer_id, - num_layers=( - config.num_hidden_layers if layer_id < config.num_hidden_layers else 1 - ), # 1 is for mtp + layer_id=0 if is_nextn else layer_id, + num_layers=1 if is_nextn else config.num_hidden_layers, is_layer_sparse=self.is_moe_layer, is_previous_layer_sparse=self.is_previous_layer_sparse, is_next_layer_sparse=self.is_next_layer_sparse, diff --git a/test/registered/unit/configs/test_model_config_shapes.py b/test/registered/unit/configs/test_model_config_shapes.py index 651dc87d73..69caf6beab 100644 --- a/test/registered/unit/configs/test_model_config_shapes.py +++ b/test/registered/unit/configs/test_model_config_shapes.py @@ -35,6 +35,24 @@ class TestModelConfigShapes(CustomTestCase): model_config._derive_model_shapes() return model_config + def test_bailing_nextn_draft_keeps_the_target_attention(self): + """A Bailing draft is renamed BailingMoeForCausalLMNextN: the V2.5 one is + MLA, the V2 one keeps the GQA shapes of its target.""" + gqa = _make_text_config( + architectures=["BailingMoeForCausalLMNextN"], head_dim=128 + ) + self.assertEqual(self._derive_shapes(gqa).attention_arch, AttentionArch.MHA) + mla = _make_text_config( + architectures=["BailingMoeForCausalLMNextN"], + head_dim=192, + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + rope_scaling=None, + ) + self.assertEqual(self._derive_shapes(mla).attention_arch, AttentionArch.MLA) + def test_optional_head_dims_default_when_none(self): text_config = _make_text_config( head_dim=None, diff --git a/test/registered/unit/models/test_last_layer_communicator.py b/test/registered/unit/models/test_last_layer_communicator.py index f88af802a1..2de04dd971 100644 --- a/test/registered/unit/models/test_last_layer_communicator.py +++ b/test/registered/unit/models/test_last_layer_communicator.py @@ -158,20 +158,37 @@ CASES = { } -def is_last_layer_passed(case, num_layers, layer_id, **kwargs): +def build(case, num_layers, layer_id, config=None, **kwargs): + """Construct one decoder layer with its submodules stubbed. Returns the + LayerCommunicator kwargs, the LayerScatterModes.init_new kwargs, and the + names of the stubbed submodules it built.""" module_name, class_name, make_config, stubs, _ = CASES[case] module = import_model(module_name) communicator = MagicMock() - patches = dict(LayerCommunicator=communicator, LayerScatterModes=MagicMock()) - patches.update({name: stub for name in stubs}) + scatter_modes = MagicMock() + built = [] + + def recording_stub(name): + def make(*args, **kwargs): + built.append(name) + return StubModule() + + return make + + patches = dict(LayerCommunicator=communicator, LayerScatterModes=scatter_modes) + patches.update({name: recording_stub(name) for name in stubs}) if hasattr(module, "get_parallel"): patches["get_parallel"] = lambda: PARALLEL with patch.multiple(module, **patches): getattr(module, class_name)( - make_config(num_layers), layer_id=layer_id, **kwargs + config or make_config(num_layers), layer_id=layer_id, **kwargs ) communicator.assert_called_once() - return communicator.call_args.kwargs.get("is_last_layer", False) + return communicator.call_args.kwargs, scatter_modes.init_new.call_args.kwargs, built + + +def is_last_layer_passed(case, num_layers, layer_id, **kwargs): + return build(case, num_layers, layer_id, **kwargs)[0].get("is_last_layer", False) class TestLastLayerCommunicator(CustomTestCase): @@ -198,6 +215,42 @@ class TestLastLayerCommunicator(CustomTestCase): is_last_layer_passed(case, num_layers, layer_id, is_nextn=True) ) + def test_draft_model_layer_is_planned_as_a_one_layer_model(self): + """The layout plan of a NextN / MTP draft layer treats it as the first and + the last layer, so it takes the model's input layout and returns the + model's output layout even when its MLP runs on SCATTERED tokens.""" + for case, (*_, draft) in CASES.items(): + if draft is None: + continue + num_layers, layer_id = draft + with self.subTest(case=case): + _, planned, _ = build(case, num_layers, layer_id, is_nextn=True) + self.assertEqual((planned["layer_id"], planned["num_layers"]), (0, 1)) + + def test_bailing_draft_layer_builds_an_moe(self): + """The Bailing V2 NextN checkpoint holds expert weights, so the NextN + layer builds the sparse MoE block, also when the model's first layers + are dense.""" + config = bailing_moe_config(NUM_LAYERS) + config.first_k_dense_replace = 1 + _, planned, built = build( + "bailing_moe", NUM_LAYERS, 0, config=config, is_nextn=True + ) + self.assertIn("BailingMoESparseMoeBlock", built) + self.assertNotIn("BailingMoEMLP", built) + self.assertTrue(planned["is_layer_sparse"]) + + def test_bailing_hybrid_draft_layer_is_planned_as_sparse(self): + """The Bailing hybrid NextN layer builds an MoE, so its layout plan is + that of a sparse layer, also when the model's first layers are dense.""" + config = bailing_hybrid_config(NUM_LAYERS) + config.first_k_dense_replace = 1 + _, planned, built = build( + "bailing_moe_linear", NUM_LAYERS, 0, config=config, is_nextn=True + ) + self.assertIn("BailingMoE", built) + self.assertTrue(planned["is_layer_sparse"]) + if __name__ == "__main__": unittest.main()