diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 1abd6da506..8788dd7fdc 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -863,6 +863,18 @@ class NemotronHModel(nn.Module): self.norm_f = PPMissingLayer(return_tuple=True) self.layers_to_capture: set[int] = set() + def _capture_hidden_states(self, hidden_states, residual, boundary_idx): + if ( + is_dp_attention_enabled() + and residual is not None + and boundary_idx > 0 + and is_attn_layer(self.config.hybrid_override_pattern[boundary_idx - 1]) + ): + # Reduce a copy so the next layer still receives a TP partial. + hidden_states = attn_tp_all_reduce(hidden_states.clone()) + # Later norms update the input residual in place. + return hidden_states.clone() if residual is None else hidden_states + residual + def forward( self, input_ids: torch.Tensor, @@ -891,7 +903,7 @@ class NemotronHModel(nn.Module): hidden_states = tensor_model_parallel_all_reduce(hidden_states) hidden_states._sglang_needs_allreduce_fusion = False aux_hidden_states.append( - hidden_states if residual is None else hidden_states + residual + self._capture_hidden_states(hidden_states, residual, i) ) layer = self.layers[i] if not isinstance(layer, Layers): @@ -913,7 +925,7 @@ class NemotronHModel(nn.Module): hidden_states = tensor_model_parallel_all_reduce(hidden_states) hidden_states._sglang_needs_allreduce_fusion = False aux_hidden_states.append( - hidden_states if residual is None else hidden_states + residual + self._capture_hidden_states(hidden_states, residual, self.end_layer) ) hidden_states, _ = self.norm_f(hidden_states, residual) if aux_hidden_states: diff --git a/test/registered/unit/models/test_nemotron_h_aux_capture.py b/test/registered/unit/models/test_nemotron_h_aux_capture.py new file mode 100644 index 0000000000..a96faae2ef --- /dev/null +++ b/test/registered/unit/models/test_nemotron_h_aux_capture.py @@ -0,0 +1,131 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from torch import nn + +from sglang.srt.layers import communicator as comm +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.models import nemotron_h as model +from sglang.srt.models.nemotron_h_utils import make_layer_communicator +from sglang.srt.runtime_context import get_context, get_flags, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _rms(x): + return x * torch.rsqrt(x.square().mean(-1, keepdim=True) + 1e-5) + + +class _Norm(nn.Module): + def forward(self, x, residual=None, post_residual_addition=None): + if residual is None: + return _rms(x) + residual.add_(x) + return _rms(residual), residual + + +class _Mixer(nn.Module): + def __init__(self, scale): + super().__init__() + self.scale = scale + + def forward(self, hidden_states, **kwargs): + return hidden_states * self.scale + + +def _build(pattern, tp, dp_enabled, capture): + instance = model.NemotronHModel.__new__(model.NemotronHModel) + nn.Module.__init__(instance) + instance.config = SimpleNamespace(hybrid_override_pattern=pattern) + instance.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True) + instance.start_layer, instance.end_layer = 0, len(pattern) + instance.norm_f = _Norm() + instance.layers_to_capture = set(range(len(pattern) + 1)) if capture else set() + layers = [] + for i, kind in enumerate(pattern): + cls = model.ALL_DECODER_LAYER_TYPES[kind] + layer = cls.__new__(cls) + nn.Module.__init__(layer) + layer.norm = _Norm() + layer.prev_layer_is_attn = i > 0 and pattern[i - 1] in "M*" + layer.layer_communicator = make_layer_communicator( + layer.norm, for_attn=kind in "M*", is_last_layer=i == len(pattern) - 1 + ) + layer.mixer = _Mixer( + (0.5 / tp if dp_enabled else 0.5) if kind in "M*" else 0.25 + ) + if kind == "M": + layer._forward_mamba = lambda h, batch, mixer=layer.mixer: mixer(h) + layers.append(layer) + instance.layers = nn.ModuleList(layers) + return instance + + +class TestNemotronAuxCapture(CustomTestCase): + def test_capture_reduces_only_its_snapshot(self): + """Each auxiliary snapshot equals the full hidden state at its boundary, + also under DP attention and after later norms update the residual in place.""" + for pattern in ("*-", "M-", "**-", "*"): + for dp_enabled, tp in ((True, 2), (True, 1), (False, 2)): + with self.subTest(pattern=pattern, dp_enabled=dp_enabled, tp=tp): + self._check(pattern, dp_enabled, tp) + + def _check(self, pattern, dp_enabled, tp): + def reduce_in_place(x): + return x.mul_(tp) + + group = SimpleNamespace(all_reduce=reduce_in_place) + batch = SimpleNamespace( + input_ids=torch.zeros(2, dtype=torch.long), + forward_mode=ForwardMode.DECODE, + global_num_token_non_padded_cpu=2, + ) + inputs = torch.tensor([[0.3, -0.5, 0.7, 1.1], [-0.4, 0.9, 0.2, -0.6]]) + with ( + get_context().override_server_args( + tp_size=tp, enable_dp_attention=dp_enabled + ), + get_flags().dp.override(enabled=dp_enabled), + get_parallel().override( + attn_tp_group=group, + launch_world_rank=0, + tp_rank=0, + tp_size=tp, + attn_tp_rank=0, + attn_tp_size=tp, + attn_dp_rank=0, + attn_dp_size=1, + attn_cp_rank=0, + attn_cp_size=1, + moe_tp_rank=0, + moe_tp_size=tp, + moe_ep_rank=0, + moe_ep_size=1, + moe_dp_rank=0, + moe_dp_size=1, + ), + patch.object(comm, "get_moe_cp_size", return_value=1), + patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=False), + patch.object(comm, "apply_aiter_all_reduce_fusion", return_value=False), + ): + baseline = _build(pattern, tp, dp_enabled, False)( + batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone() + ) + output, snapshots = _build(pattern, tp, dp_enabled, True)( + batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone() + ) + torch.testing.assert_close(output, baseline) + expected = [inputs.clone()] + for kind in pattern: + value = expected[-1] + expected.append(value + _rms(value) * (0.5 if kind in "M*" else 0.25)) + for i in (*range(1, len(expected)), 0): + torch.testing.assert_close(snapshots[i], expected[i], msg=f"Boundary {i}") + + +if __name__ == "__main__": + unittest.main()