[Refactor] Run Nemotron-H DP attention through the standard layer communicator (#40867)

This commit is contained in:
Cheng Wan
2026-09-23 14:40:33 -07:00
committed by GitHub
parent 379e8f916b
commit 701cf7e47e
11 changed files with 318 additions and 301 deletions
@@ -157,11 +157,11 @@ class MambaAttnBackendBase(AttentionBackend):
logical_num_tokens = None
track_mask_indices = None
# Padded rows carry req-pool row 0, which maps to the reserved mamba
# padding slot 0 in both the static and the unified pool.
mamba_cache_indices = self.req_to_token_pool.get_mamba_indices(
forward_batch.req_pool_indices
)
# Translate virtual->physical BEFORE the padding sentinel below, so the
# gather reads only real ids; padded rows are then poisoned to -1 (skipped).
mamba_cache_indices = self._translate_mamba_indices(mamba_cache_indices)
if forward_batch.mamba_track_indices is not None:
forward_batch.mamba_track_indices = self._translate_mamba_indices(
@@ -182,11 +182,6 @@ class MambaAttnBackendBase(AttentionBackend):
forward_batch.mamba_track_mask is not None
and forward_batch.mamba_track_mask.any()
)
_real_bs = forward_batch._original_batch_size
if _real_bs is not None and _real_bs < mamba_cache_indices.shape[0]:
mamba_cache_indices = mamba_cache_indices.clone()
mamba_cache_indices[_real_bs:] = -1
replayssm_write_pos = None
replayssm_force_flush = None
if forward_batch.forward_mode.is_decode_or_idle():
@@ -209,8 +204,8 @@ class MambaAttnBackendBase(AttentionBackend):
)
if write_pos_buf is not None:
slots = mamba_cache_indices.to(torch.long)
# Padded rows carry slot == -1; clamp the gather in-bounds (kernel
# zeroes padded rows via state_idx < 0).
# Padded rows point at the reserved padding slot 0; the clamp keeps
# a -1 sentinel in bounds (the kernel zeroes rows with state_idx < 0).
safe_slots = slots.clamp(min=0)
replayssm_write_pos = write_pos_buf[safe_slots].clone()
L = mamba_pool.linear_replayssm_cache_len
@@ -227,7 +222,7 @@ class MambaAttnBackendBase(AttentionBackend):
device=self.device, dtype=torch.int32
)
# Advance only valid slots, scattered over unique slots (dup-index
# race; padded rows clamp to 0); a forced flush -> next write_pos 0.
# race; padded rows share slot 0); a forced flush -> next write_pos 0.
valid_mask = slots >= 0
valid_slots = slots[valid_mask]
if valid_slots.numel() > 0:
@@ -211,6 +211,7 @@ class MambaMixer2(torch.nn.Module):
activation: str = "silu",
use_rms_norm: bool = True,
quant_config: Optional[QuantizationConfig] = None,
reduce_results: Optional[bool] = None,
prefix: str = "",
):
super().__init__()
@@ -421,6 +422,9 @@ class MambaMixer2(torch.nn.Module):
set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
# By default a layer communicator reduces the output under DP attention.
if reduce_results is None:
reduce_results = not is_dp_attention_enabled()
self.out_proj = RowParallelLinear(
intermediate_size,
hidden_size,
@@ -429,7 +433,8 @@ class MambaMixer2(torch.nn.Module):
quant_config=quant_config,
tp_rank=self.tp_rank,
tp_size=self.tp_size,
reduce_results=not is_dp_attention_enabled(),
reduce_results=reduce_results,
use_dp_attention_reduce=reduce_results and is_dp_attention_enabled(),
prefix=f"{prefix}.out_proj",
)
@@ -545,6 +550,9 @@ class MambaMixer2(torch.nn.Module):
dtype=hidden_states.dtype,
device=hidden_states.device,
)
# Rows past the batch's tokens are DP padding; keep them finite.
if num_actual_tokens < preallocated_ssm_out.shape[0]:
preallocated_ssm_out[num_actual_tokens:].zero_()
preallocated_ssm_out_active = preallocated_ssm_out[:num_actual_tokens]
preallocated_ssm_out_p, preallocated_ssm_out_d = torch.split(
preallocated_ssm_out_active,
@@ -196,7 +196,6 @@ class Mamba2Metadata(ForwardMetadata):
*,
is_target_verify: bool,
draft_token_num: int,
num_decodes: Optional[int] = None,
) -> "Mamba2Metadata":
"""This path is run during CUDA graph capture, i.e. decode only, so `num_prefills` is 0"""
return Mamba2Metadata(
@@ -215,7 +214,7 @@ class Mamba2Metadata(ForwardMetadata):
track_ssm_end_locs=forward_metadata.track_ssm_end_locs,
track_ssm_recompute_dst=forward_metadata.track_ssm_recompute_dst,
has_mamba_track_mask=forward_metadata.has_mamba_track_mask,
num_decodes=len(seq_lens) if num_decodes is None else num_decodes,
num_decodes=len(seq_lens),
num_prefills=0,
num_prefill_tokens=0,
is_target_verify=is_target_verify,
@@ -236,15 +235,11 @@ class Mamba2Metadata(ForwardMetadata):
if forward_batch.spec_info is not None
else 1
)
num_decodes = getattr(forward_batch, "_original_batch_size", None)
if num_decodes is None:
num_decodes = len(forward_batch.seq_lens)
return cls.prepare_decode(
forward_metadata,
forward_batch.seq_lens,
is_target_verify=forward_batch.forward_mode.is_target_verify(),
draft_token_num=draft_token_num,
num_decodes=num_decodes,
)
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
if extend_seq_lens_cpu is None:
@@ -255,10 +250,7 @@ class Mamba2Metadata(ForwardMetadata):
num_prefill_tokens = int(sum(extend_seq_lens_cpu))
else:
num_prefill_tokens = int(forward_batch.extend_num_tokens)
batch_size = getattr(forward_batch, "_original_batch_size", None)
if batch_size is None:
batch_size = len(forward_batch.seq_lens)
num_decodes = max(0, batch_size - num_prefills)
num_decodes = len(forward_batch.seq_lens) - num_prefills
context_lens_tensor = forward_batch.extend_prefix_lens
assert context_lens_tensor is not None
has_initial_states = context_lens_tensor > 0
@@ -160,6 +160,11 @@ def _linear_attention_with_output_impl(
real_num_tokens = min(
forward_batch.global_num_token_non_padded_cpu, mixed_qkv.shape[0]
)
if real_num_tokens == 0:
# A fully masked batch (an idle DP rank) needs no attention or state
# update, and GDN prefill kernels reject an empty varlen batch.
output.zero_()
return
original_out_cache_loc = forward_batch.out_cache_loc
# Keep the original ForwardBatch object and only narrow cache locations for
@@ -1591,9 +1591,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
or self.forward_mode.is_draft_extend_v2()
or self.forward_mode.is_idle()
):
# Mamba-hybrid families need the fabricated-row idle conversion
# below; this includes their MTP draft workers, whose mamba-less
# "*E" pattern makes mambaish_config return None.
# TARGET_VERIFY counts as extend; hybrid-SSM verify batches keep their
# verify layout. MTP draft workers ("*E", no mamba layer) count as hybrid.
hybrid_ssm = mambaish_config(model_runner.model_config) is not None or (
model_runner.is_draft_worker
and getattr(
@@ -1617,16 +1616,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self._original_forward_mode = self.forward_mode
self.forward_mode = ForwardMode.EXTEND
# Fabricate a single dummy request covering num_tokens for an
# empty (idle) rank. Hybrid-SSM families always take this path;
# non-hybrid ranks reach it once MAX_LEN is forced for the
# prefill breakable CUDA graph (idle + prefill), which needs
# empty (idle) rank. Ranks reach this once MAX_LEN is forced for
# the prefill breakable CUDA graph (idle + prefill), which needs
# every DP rank to run the same captured shape. The `else`
# branch handles decode rows padded to a 1-token extend.
if hybrid_ssm or self.seq_lens.shape[0] == 0:
if self.seq_lens.shape[0] == 0:
dev = self.seq_lens.device
assert self.seq_lens.shape[0] == 0, (
"extend-idle conversion expects an empty rank"
)
self.extend_num_tokens = num_tokens
self.extend_seq_lens = torch.tensor(
[num_tokens], dtype=torch.int32, device=dev
@@ -1655,15 +1650,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self.extend_seq_lens_cpu = [int(num_tokens)]
self.extend_logprob_start_lens_cpu = [0]
bs = self.batch_size = 1
# Keep idle non-hybrid fabricated rows masked by default.
# Hybrid-SSM needs the real count for its state update.
mask_dummy_tokens = (
not hybrid_ssm and self._original_forward_mode.is_idle()
)
# Bump the GLOBAL scalar; the LOCAL count is derived from it
# downstream. (global_num_token_non_padded is None unless
# Keep idle fabricated rows masked. Bump the GLOBAL scalar;
# the LOCAL count is derived from it downstream.
# (global_num_token_non_padded is None unless
# moe_ep_size > 1.)
if mask_dummy_tokens:
if self._original_forward_mode.is_idle():
if self.global_num_token_non_padded is not None:
self.global_num_token_non_padded.fill_(0)
self.global_num_token_non_padded_cpu = 0
+108 -189
View File
@@ -81,11 +81,9 @@ from sglang.srt.model_loader.weight_utils import (
replace_substrings,
)
from sglang.srt.models.nemotron_h_utils import (
get_real_num_tokens,
input_norm_maybe_fuse_allreduce,
feeds_mlp_layer,
is_attn_layer,
make_layer_communicator,
pad_to_original_num_tokens,
)
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
@@ -383,7 +381,24 @@ class NemotronHMoE(nn.Module):
class NemotronHMLPLikeDecoderLayer(nn.Module):
"""Shared forward for the dense-MLP / MoE decoder layers."""
"""MLP half of a decoder layer. A preceding Mamba / attention layer hands over
its unreduced output, as between the two halves of a standard decoder layer."""
def _init_layer_communicator(
self, config: NemotronHConfig, layer_idx: int, *, is_sparse: bool
) -> None:
pattern = config.hybrid_override_pattern
self.input_is_reduced = layer_idx == 0 or not is_attn_layer(
pattern[layer_idx - 1]
)
self.feeds_mlp_layer = feeds_mlp_layer(pattern, layer_idx)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=False,
allow_reduce_scatter=True,
is_sparse=is_sparse,
is_last_layer=layer_idx == len(pattern) - 1,
)
def forward(
self,
@@ -392,44 +407,36 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
if is_dp_attention_enabled():
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
):
hidden_states = self.mixer.forward(hidden_states)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
return hidden_states, residual
hidden_states, residual = input_norm_maybe_fuse_allreduce(
self.norm, hidden_states, residual
if self.input_is_reduced:
# Fold the reduced input into the residual, leaving prepare_mlp a
# zero update to reduce.
residual = hidden_states if residual is None else hidden_states + residual
hidden_states = torch.zeros_like(hidden_states)
hidden_states, residual = self.layer_communicator.prepare_mlp(
hidden_states, residual, forward_batch
)
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
# prepare_mlp ignores a deferred reduction, so only defer into a
# Mamba / attention layer.
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
not self.feeds_mlp_layer
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
with get_forward().scoped(
fuse_mlp_allreduce=fuse_mlp_allreduce,
mlp_reduce_scatter=mlp_reduce_scatter,
):
hidden_states = self.mixer.forward(hidden_states)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
else:
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
return hidden_states, residual
@@ -464,12 +471,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=False,
allow_reduce_scatter=True,
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
)
self._init_layer_communicator(config, layer_idx, is_sparse=False)
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
@@ -492,34 +494,50 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=False,
allow_reduce_scatter=True,
is_sparse=True,
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
)
self._init_layer_communicator(config, layer_idx, is_sparse=True)
class NemotronHAttnLikeDecoderLayer(nn.Module):
"""Shared DP-attention input prep for the Mamba / full-attention layers."""
"""Attention half of a decoder layer. Before an MLP layer the mixer leaves its
output unreduced for prepare_mlp; otherwise the mixer reduces it itself."""
def _set_prev_layer_is_attn(self, config: NemotronHConfig, layer_idx: int) -> None:
self.prev_layer_is_attn = layer_idx > 0 and is_attn_layer(
config.hybrid_override_pattern[layer_idx - 1]
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
self.feeds_mlp_layer = feeds_mlp_layer(
config.hybrid_override_pattern, layer_idx
)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=True,
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
)
def _dp_attn_input(
def forward(
self,
*,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if self.prev_layer_is_attn and residual is not None:
hidden_states = attn_tp_all_reduce(hidden_states)
return self.layer_communicator.prepare_attn(
) -> tuple[torch.Tensor, torch.Tensor]:
hidden_states, residual = self.layer_communicator.prepare_attn(
hidden_states, residual, forward_batch
)
if forward_batch.forward_mode.is_idle():
return hidden_states, residual
fuse_mlp_allreduce = (
not self.feeds_mlp_layer
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
skip_reduce = self.feeds_mlp_layer or fuse_mlp_allreduce
with get_forward().scoped(fuse_mlp_allreduce=skip_reduce):
hidden_states = self._forward_mixer(
hidden_states, forward_batch, skip_reduce
)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states, residual
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
@@ -542,16 +560,12 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
rms_norm_eps=config.layer_norm_epsilon,
activation=config.mamba_hidden_act,
quant_config=quant_config,
reduce_results=True,
prefix=f"{prefix}.mixer",
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=True,
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
)
self._set_prev_layer_is_attn(config, layer_idx)
self._init_layer_communicator(config, layer_idx)
def _forward_mamba(
self,
@@ -559,15 +573,10 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
forward_batch: ForwardBatch,
) -> torch.Tensor:
"""Core Mamba forward logic, called directly or via split op."""
original_num_tokens = hidden_states.shape[0]
if forward_batch.forward_mode.is_extend():
real_num_tokens = get_real_num_tokens(hidden_states, forward_batch)
if real_num_tokens < original_num_tokens:
hidden_states = hidden_states[:real_num_tokens]
attn_backend = get_attn_backend()
assert isinstance(attn_backend, HybridLinearAttnBackend)
assert isinstance(attn_backend.linear_attn_backend, Mamba2AttnBackend)
output = attn_backend.linear_attn_backend.forward(
return attn_backend.linear_attn_backend.forward(
mixer=self.mixer,
layer_id=self.layer_id,
hidden_states=hidden_states,
@@ -575,61 +584,26 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
forward_batch=forward_batch,
use_triton_causal_conv=True,
)
return pad_to_original_num_tokens(output, original_num_tokens)
def forward(
def _forward_mixer(
self,
*,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
if is_dp_attention_enabled():
hidden_states, residual = self._dp_attn_input(
hidden_states, residual, forward_batch
skip_reduce: bool,
) -> torch.Tensor:
if is_in_breakable_cuda_graph():
output = torch.empty_like(hidden_states)
breakable_nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, skip_reduce
)
if get_real_num_tokens(hidden_states, forward_batch) == 0:
return torch.zeros_like(hidden_states), residual
if is_in_breakable_cuda_graph():
output = torch.empty_like(hidden_states)
breakable_nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, False
)
elif is_in_tc_piecewise_cuda_graph():
output = torch.empty_like(hidden_states)
nemotron_mamba2_with_output(hidden_states, output, self.layer_id, False)
else:
output = self._forward_mamba(hidden_states, forward_batch)
return output, residual
hidden_states, residual = input_norm_maybe_fuse_allreduce(
self.norm, hidden_states, residual
)
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
return output
if is_in_tc_piecewise_cuda_graph():
output = torch.empty_like(hidden_states)
nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, skip_reduce
)
)
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
if is_in_breakable_cuda_graph():
output = torch.empty_like(hidden_states)
breakable_nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, fuse_mlp_allreduce
)
elif is_in_tc_piecewise_cuda_graph():
output = torch.empty_like(hidden_states)
nemotron_mamba2_with_output(
hidden_states, output, self.layer_id, fuse_mlp_allreduce
)
else:
output = self._forward_mamba(hidden_states, forward_batch)
if fuse_mlp_allreduce:
output._sglang_needs_allreduce_fusion = True
return output, residual
return output
return self._forward_mamba(hidden_states, forward_batch)
class NemotronHAttention(nn.Module):
@@ -683,7 +657,7 @@ class NemotronHAttention(nn.Module):
quant_config=quant_config,
tp_rank=tp_rank,
tp_size=tp_size,
reduce_results=not is_dp_attention_enabled(),
use_dp_attention_reduce=is_dp_attention_enabled(),
prefix=f"{prefix}.o_proj",
)
@@ -703,38 +677,9 @@ class NemotronHAttention(nn.Module):
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
if not is_dp_attention_enabled():
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
attn_output = self.attn.forward(q, k, v, forward_batch)
output, _ = self.o_proj(attn_output)
return output
padded_shape = hidden_states.shape[0]
real_tokens = get_real_num_tokens(hidden_states, forward_batch)
has_padding = real_tokens < padded_shape
keep_q_padded = (
forward_batch.forward_mode.is_decode()
or forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_idle()
or forward_batch._original_forward_mode is not None
)
original_out_cache_loc = forward_batch.out_cache_loc
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
if has_padding and real_tokens > 0:
k, v = k[:real_tokens], v[:real_tokens]
if original_out_cache_loc is not None:
forward_batch.out_cache_loc = original_out_cache_loc[:real_tokens]
if not keep_q_padded:
q = q[:real_tokens]
attn_output = self.attn.forward(
q, k, v, forward_batch, save_kv_cache=real_tokens > 0
)
forward_batch.out_cache_loc = original_out_cache_loc
attn_output = pad_to_original_num_tokens(attn_output, padded_shape)
attn_output = self.attn.forward(q, k, v, forward_batch)
output, _ = self.o_proj(attn_output)
return output
@@ -758,48 +703,18 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self.layer_communicator = make_layer_communicator(
self.norm,
for_attn=True,
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
)
self._set_prev_layer_is_attn(config, layer_idx)
self._init_layer_communicator(config, layer_idx)
def forward(
def _forward_mixer(
self,
*,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor]:
if is_dp_attention_enabled():
hidden_states, residual = self._dp_attn_input(
hidden_states, residual, forward_batch
)
hidden_states = self.mixer.forward(
hidden_states=hidden_states, forward_batch=forward_batch
)
return hidden_states, residual
hidden_states, residual = input_norm_maybe_fuse_allreduce(
self.norm, hidden_states, residual
skip_reduce: bool,
) -> torch.Tensor:
return self.mixer.forward(
hidden_states=hidden_states, forward_batch=forward_batch
)
fuse_mlp_allreduce = (
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
forward_batch
)
)
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
hidden_states = self.mixer.forward(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
if fuse_mlp_allreduce:
hidden_states._sglang_needs_allreduce_fusion = True
return hidden_states, residual
Layers = (
NemotronHAttentionDecoderLayer,
@@ -864,13 +779,14 @@ class NemotronHModel(nn.Module):
self.layers_to_capture: set[int] = set()
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
pattern = self.config.hybrid_override_pattern
if (
is_dp_attention_enabled()
and residual is not None
residual is not None
and boundary_idx > 0
and is_attn_layer(self.config.hybrid_override_pattern[boundary_idx - 1])
and is_attn_layer(pattern[boundary_idx - 1])
and feeds_mlp_layer(pattern, boundary_idx - 1)
):
# Reduce a copy so the next layer still receives a TP partial.
# Reduce a copy so the MLP layer still receives a TP partial.
hidden_states = attn_tp_all_reduce(hidden_states.clone())
# Later norms update the input residual in place.
return hidden_states.clone() if residual is None else hidden_states + residual
@@ -927,7 +843,10 @@ class NemotronHModel(nn.Module):
aux_hidden_states.append(
self._capture_hidden_states(hidden_states, residual, self.end_layer)
)
hidden_states, _ = self.norm_f(hidden_states, residual)
if residual is None:
hidden_states = self.norm_f(hidden_states)
else:
hidden_states, _ = self.norm_f(hidden_states, residual)
if aux_hidden_states:
return hidden_states, aux_hidden_states
return hidden_states
+6 -55
View File
@@ -1,19 +1,15 @@
"""DP-attention helpers for the Nemotron-H model."""
"""Layer-communication helpers for the Nemotron-H model."""
import torch
from torch import nn
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
apply_flashinfer_allreduce_fusion,
)
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
ATTN_LAYERS = (MAMBA, ATTENTION)
@@ -22,32 +18,9 @@ def is_attn_layer(layer_type: str) -> bool:
return layer_type in ATTN_LAYERS
def get_real_num_tokens(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> int:
"""Number of real (non DP-padding) rows in ``hidden_states``."""
real_tokens = hidden_states.shape[0]
if forward_batch.global_num_token_non_padded_cpu is not None:
real_tokens = min(
real_tokens, int(forward_batch.global_num_token_non_padded_cpu)
)
if (
forward_batch.forward_mode.is_extend()
and not forward_batch.forward_mode.is_mixed()
and forward_batch.extend_seq_lens_cpu is not None
):
real_tokens = min(real_tokens, int(sum(forward_batch.extend_seq_lens_cpu)))
return real_tokens
def pad_to_original_num_tokens(
output: torch.Tensor, original_num_tokens: int
) -> torch.Tensor:
if output.shape[0] == original_num_tokens:
return output
padded = output.new_empty((original_num_tokens, *output.shape[1:]))
padded[: output.shape[0]] = output
return padded
def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool:
next_idx = layer_idx + 1
return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx])
def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes:
@@ -77,31 +50,9 @@ def make_layer_communicator(
layer_scatter_modes=_build_layer_scatter_modes(is_sparse),
input_layernorm=layer_norm if for_attn else nn.Identity(),
post_attention_layernorm=nn.Identity() if for_attn else layer_norm,
# With attention TP > 1, the default gather adds the residual to one
# rank's partial in bf16 before the cross-rank sum.
force_layernorm_before_dp_gather=True,
allow_reduce_scatter=allow_reduce_scatter,
is_last_layer=is_last_layer,
)
def input_norm_maybe_fuse_allreduce(
norm: RMSNorm,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is not None and getattr(
hidden_states, "_sglang_needs_allreduce_fusion", False
):
if apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) and hasattr(
norm, "forward_with_allreduce_fusion"
):
return norm.forward_with_allreduce_fusion(
hidden_states, residual, use_attn_tp_group=False
)
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
return norm(hidden_states, residual)
if residual is None:
residual = hidden_states
hidden_states = norm(hidden_states)
return hidden_states, residual
return norm(hidden_states, residual)