[Fix] Capture complete Nemotron auxiliary hidden states (#40801)
This commit is contained in:
@@ -863,6 +863,18 @@ class NemotronHModel(nn.Module):
|
||||
self.norm_f = PPMissingLayer(return_tuple=True)
|
||||
self.layers_to_capture: set[int] = set()
|
||||
|
||||
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
|
||||
if (
|
||||
is_dp_attention_enabled()
|
||||
and residual is not None
|
||||
and boundary_idx > 0
|
||||
and is_attn_layer(self.config.hybrid_override_pattern[boundary_idx - 1])
|
||||
):
|
||||
# Reduce a copy so the next layer still receives a TP partial.
|
||||
hidden_states = attn_tp_all_reduce(hidden_states.clone())
|
||||
# Later norms update the input residual in place.
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -891,7 +903,7 @@ class NemotronHModel(nn.Module):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
self._capture_hidden_states(hidden_states, residual, i)
|
||||
)
|
||||
layer = self.layers[i]
|
||||
if not isinstance(layer, Layers):
|
||||
@@ -913,7 +925,7 @@ class NemotronHModel(nn.Module):
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states._sglang_needs_allreduce_fusion = False
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
self._capture_hidden_states(hidden_states, residual, self.end_layer)
|
||||
)
|
||||
hidden_states, _ = self.norm_f(hidden_states, residual)
|
||||
if aux_hidden_states:
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.models import nemotron_h as model
|
||||
from sglang.srt.models.nemotron_h_utils import make_layer_communicator
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def _rms(x):
|
||||
return x * torch.rsqrt(x.square().mean(-1, keepdim=True) + 1e-5)
|
||||
|
||||
|
||||
class _Norm(nn.Module):
|
||||
def forward(self, x, residual=None, post_residual_addition=None):
|
||||
if residual is None:
|
||||
return _rms(x)
|
||||
residual.add_(x)
|
||||
return _rms(residual), residual
|
||||
|
||||
|
||||
class _Mixer(nn.Module):
|
||||
def __init__(self, scale):
|
||||
super().__init__()
|
||||
self.scale = scale
|
||||
|
||||
def forward(self, hidden_states, **kwargs):
|
||||
return hidden_states * self.scale
|
||||
|
||||
|
||||
def _build(pattern, tp, dp_enabled, capture):
|
||||
instance = model.NemotronHModel.__new__(model.NemotronHModel)
|
||||
nn.Module.__init__(instance)
|
||||
instance.config = SimpleNamespace(hybrid_override_pattern=pattern)
|
||||
instance.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
|
||||
instance.start_layer, instance.end_layer = 0, len(pattern)
|
||||
instance.norm_f = _Norm()
|
||||
instance.layers_to_capture = set(range(len(pattern) + 1)) if capture else set()
|
||||
layers = []
|
||||
for i, kind in enumerate(pattern):
|
||||
cls = model.ALL_DECODER_LAYER_TYPES[kind]
|
||||
layer = cls.__new__(cls)
|
||||
nn.Module.__init__(layer)
|
||||
layer.norm = _Norm()
|
||||
layer.prev_layer_is_attn = i > 0 and pattern[i - 1] in "M*"
|
||||
layer.layer_communicator = make_layer_communicator(
|
||||
layer.norm, for_attn=kind in "M*", is_last_layer=i == len(pattern) - 1
|
||||
)
|
||||
layer.mixer = _Mixer(
|
||||
(0.5 / tp if dp_enabled else 0.5) if kind in "M*" else 0.25
|
||||
)
|
||||
if kind == "M":
|
||||
layer._forward_mamba = lambda h, batch, mixer=layer.mixer: mixer(h)
|
||||
layers.append(layer)
|
||||
instance.layers = nn.ModuleList(layers)
|
||||
return instance
|
||||
|
||||
|
||||
class TestNemotronAuxCapture(CustomTestCase):
|
||||
def test_capture_reduces_only_its_snapshot(self):
|
||||
"""Each auxiliary snapshot equals the full hidden state at its boundary,
|
||||
also under DP attention and after later norms update the residual in place."""
|
||||
for pattern in ("*-", "M-", "**-", "*"):
|
||||
for dp_enabled, tp in ((True, 2), (True, 1), (False, 2)):
|
||||
with self.subTest(pattern=pattern, dp_enabled=dp_enabled, tp=tp):
|
||||
self._check(pattern, dp_enabled, tp)
|
||||
|
||||
def _check(self, pattern, dp_enabled, tp):
|
||||
def reduce_in_place(x):
|
||||
return x.mul_(tp)
|
||||
|
||||
group = SimpleNamespace(all_reduce=reduce_in_place)
|
||||
batch = SimpleNamespace(
|
||||
input_ids=torch.zeros(2, dtype=torch.long),
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
global_num_token_non_padded_cpu=2,
|
||||
)
|
||||
inputs = torch.tensor([[0.3, -0.5, 0.7, 1.1], [-0.4, 0.9, 0.2, -0.6]])
|
||||
with (
|
||||
get_context().override_server_args(
|
||||
tp_size=tp, enable_dp_attention=dp_enabled
|
||||
),
|
||||
get_flags().dp.override(enabled=dp_enabled),
|
||||
get_parallel().override(
|
||||
attn_tp_group=group,
|
||||
launch_world_rank=0,
|
||||
tp_rank=0,
|
||||
tp_size=tp,
|
||||
attn_tp_rank=0,
|
||||
attn_tp_size=tp,
|
||||
attn_dp_rank=0,
|
||||
attn_dp_size=1,
|
||||
attn_cp_rank=0,
|
||||
attn_cp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_tp_size=tp,
|
||||
moe_ep_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_dp_rank=0,
|
||||
moe_dp_size=1,
|
||||
),
|
||||
patch.object(comm, "get_moe_cp_size", return_value=1),
|
||||
patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=False),
|
||||
patch.object(comm, "apply_aiter_all_reduce_fusion", return_value=False),
|
||||
):
|
||||
baseline = _build(pattern, tp, dp_enabled, False)(
|
||||
batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone()
|
||||
)
|
||||
output, snapshots = _build(pattern, tp, dp_enabled, True)(
|
||||
batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone()
|
||||
)
|
||||
torch.testing.assert_close(output, baseline)
|
||||
expected = [inputs.clone()]
|
||||
for kind in pattern:
|
||||
value = expected[-1]
|
||||
expected.append(value + _rms(value) * (0.5 if kind in "M*" else 0.25))
|
||||
for i in (*range(1, len(expected)), 0):
|
||||
torch.testing.assert_close(snapshots[i], expected[i], msg=f"Boundary {i}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user