diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 10065f48e6..5f77563863 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -18,10 +18,7 @@ import torch from torch import nn from sglang.srt.configs import NemotronHConfig -from sglang.srt.layers.dp_attention import ( - attn_tp_all_reduce, - is_dp_attention_enabled, -) +from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -37,7 +34,6 @@ from sglang.srt.models.nemotron_h import ( NemotronHForCausalLM, NemotronHMoEDecoderLayer, ) -from sglang.srt.models.nemotron_h_utils import is_attn_layer from sglang.srt.models.utils import WeightsMapper from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import add_prefix @@ -142,10 +138,6 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): ) self.has_start_projections = has_start_projections self.has_end_norm = has_end_norm - _pat = config.mtp_hybrid_override_pattern - self.prev_layer_is_attn = layer_idx > 0 and is_attn_layer( - _pat[(layer_idx - 1) % len(_pat)] - ) self.layer_communicator.is_last_layer = True if has_start_projections: @@ -194,13 +186,6 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): hidden_states, dim=-1 ) - if ( - is_dp_attention_enabled() - and self.prev_layer_is_attn - and residual is not None - ): - hidden_states = attn_tp_all_reduce(hidden_states) - hidden_states, residual = super().forward( hidden_states=hidden_states, residual=residual, diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py new file mode 100644 index 0000000000..3c6a7ce85e --- /dev/null +++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py @@ -0,0 +1,93 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +from torch import nn + +from sglang.srt.layers import communicator as comm +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.models import nemotron_h_mtp +from sglang.srt.models.nemotron_h_utils import make_layer_communicator +from sglang.srt.runtime_context import get_context, get_flags, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +class _Norm(nn.Module): + def forward(self, x, residual=None): + if residual is None: + return x + return x + residual, x + residual + + +class TestNemotronMTPReduction(CustomTestCase): + def test_attention_partial_is_reduced_once(self): + """Under DP attention, the MTP MoE layer sums the attention output over the + attention TP group exactly once before the residual add.""" + for tp in (1, 2): + with self.subTest(tp=tp): + reduce = Mock(side_effect=lambda x: x * tp) + group = SimpleNamespace(all_reduce=reduce) + with ( + get_context().override_server_args( + tp_size=tp, enable_dp_attention=True + ), + get_flags().dp.override(enabled=True), + get_parallel().override( + attn_tp_group=group, + launch_world_rank=0, + tp_rank=0, + tp_size=tp, + attn_tp_rank=0, + attn_tp_size=tp, + attn_dp_rank=0, + attn_dp_size=1, + attn_cp_rank=0, + attn_cp_size=1, + moe_tp_rank=0, + moe_tp_size=tp, + moe_ep_rank=0, + moe_ep_size=1, + moe_dp_rank=0, + moe_dp_size=1, + ), + patch.object(comm, "get_moe_cp_size", return_value=1), + patch.object( + comm, "apply_flashinfer_allreduce_fusion", return_value=False + ), + patch.object( + comm, "apply_aiter_all_reduce_fusion", return_value=False + ), + ): + layer = nemotron_h_mtp.NemotronHMTPMoEDecoderLayer.__new__( + nemotron_h_mtp.NemotronHMTPMoEDecoderLayer + ) + nn.Module.__init__(layer) + layer.has_start_projections = False + layer.has_end_norm = False + layer.mixer = nn.Identity() + layer.layer_communicator = make_layer_communicator( + _Norm(), for_attn=False, is_last_layer=True + ) + partial = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + residual = torch.tensor([[7.0, 3.0], [5.0, 9.0]]) + expected = partial * tp + residual + hidden, output_residual = layer( + inputs_embeds=torch.zeros_like(partial), + hidden_states=partial, + residual=residual, + forward_batch=SimpleNamespace( + forward_mode=ForwardMode.DECODE, + input_ids=torch.zeros(2, dtype=torch.long), + ), + ) + torch.testing.assert_close(hidden, expected) + torch.testing.assert_close(output_residual, expected) + self.assertEqual(reduce.call_count, int(tp > 1)) + + +if __name__ == "__main__": + unittest.main()