[Refactor] Run Nemotron-H DP attention through the standard layer communicator (#40867)
This commit is contained in:
@@ -157,11 +157,11 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
logical_num_tokens = None
|
||||
track_mask_indices = None
|
||||
|
||||
# Padded rows carry req-pool row 0, which maps to the reserved mamba
|
||||
# padding slot 0 in both the static and the unified pool.
|
||||
mamba_cache_indices = self.req_to_token_pool.get_mamba_indices(
|
||||
forward_batch.req_pool_indices
|
||||
)
|
||||
# Translate virtual->physical BEFORE the padding sentinel below, so the
|
||||
# gather reads only real ids; padded rows are then poisoned to -1 (skipped).
|
||||
mamba_cache_indices = self._translate_mamba_indices(mamba_cache_indices)
|
||||
if forward_batch.mamba_track_indices is not None:
|
||||
forward_batch.mamba_track_indices = self._translate_mamba_indices(
|
||||
@@ -182,11 +182,6 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
forward_batch.mamba_track_mask is not None
|
||||
and forward_batch.mamba_track_mask.any()
|
||||
)
|
||||
_real_bs = forward_batch._original_batch_size
|
||||
if _real_bs is not None and _real_bs < mamba_cache_indices.shape[0]:
|
||||
mamba_cache_indices = mamba_cache_indices.clone()
|
||||
mamba_cache_indices[_real_bs:] = -1
|
||||
|
||||
replayssm_write_pos = None
|
||||
replayssm_force_flush = None
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
@@ -209,8 +204,8 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
)
|
||||
if write_pos_buf is not None:
|
||||
slots = mamba_cache_indices.to(torch.long)
|
||||
# Padded rows carry slot == -1; clamp the gather in-bounds (kernel
|
||||
# zeroes padded rows via state_idx < 0).
|
||||
# Padded rows point at the reserved padding slot 0; the clamp keeps
|
||||
# a -1 sentinel in bounds (the kernel zeroes rows with state_idx < 0).
|
||||
safe_slots = slots.clamp(min=0)
|
||||
replayssm_write_pos = write_pos_buf[safe_slots].clone()
|
||||
L = mamba_pool.linear_replayssm_cache_len
|
||||
@@ -227,7 +222,7 @@ class MambaAttnBackendBase(AttentionBackend):
|
||||
device=self.device, dtype=torch.int32
|
||||
)
|
||||
# Advance only valid slots, scattered over unique slots (dup-index
|
||||
# race; padded rows clamp to 0); a forced flush -> next write_pos 0.
|
||||
# race; padded rows share slot 0); a forced flush -> next write_pos 0.
|
||||
valid_mask = slots >= 0
|
||||
valid_slots = slots[valid_mask]
|
||||
if valid_slots.numel() > 0:
|
||||
|
||||
@@ -211,6 +211,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
activation: str = "silu",
|
||||
use_rms_norm: bool = True,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: Optional[bool] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -421,6 +422,9 @@ class MambaMixer2(torch.nn.Module):
|
||||
set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
# By default a layer communicator reduces the output under DP attention.
|
||||
if reduce_results is None:
|
||||
reduce_results = not is_dp_attention_enabled()
|
||||
self.out_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
@@ -429,7 +433,8 @@ class MambaMixer2(torch.nn.Module):
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
reduce_results=not is_dp_attention_enabled(),
|
||||
reduce_results=reduce_results,
|
||||
use_dp_attention_reduce=reduce_results and is_dp_attention_enabled(),
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
@@ -545,6 +550,9 @@ class MambaMixer2(torch.nn.Module):
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Rows past the batch's tokens are DP padding; keep them finite.
|
||||
if num_actual_tokens < preallocated_ssm_out.shape[0]:
|
||||
preallocated_ssm_out[num_actual_tokens:].zero_()
|
||||
preallocated_ssm_out_active = preallocated_ssm_out[:num_actual_tokens]
|
||||
preallocated_ssm_out_p, preallocated_ssm_out_d = torch.split(
|
||||
preallocated_ssm_out_active,
|
||||
|
||||
@@ -196,7 +196,6 @@ class Mamba2Metadata(ForwardMetadata):
|
||||
*,
|
||||
is_target_verify: bool,
|
||||
draft_token_num: int,
|
||||
num_decodes: Optional[int] = None,
|
||||
) -> "Mamba2Metadata":
|
||||
"""This path is run during CUDA graph capture, i.e. decode only, so `num_prefills` is 0"""
|
||||
return Mamba2Metadata(
|
||||
@@ -215,7 +214,7 @@ class Mamba2Metadata(ForwardMetadata):
|
||||
track_ssm_end_locs=forward_metadata.track_ssm_end_locs,
|
||||
track_ssm_recompute_dst=forward_metadata.track_ssm_recompute_dst,
|
||||
has_mamba_track_mask=forward_metadata.has_mamba_track_mask,
|
||||
num_decodes=len(seq_lens) if num_decodes is None else num_decodes,
|
||||
num_decodes=len(seq_lens),
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
is_target_verify=is_target_verify,
|
||||
@@ -236,15 +235,11 @@ class Mamba2Metadata(ForwardMetadata):
|
||||
if forward_batch.spec_info is not None
|
||||
else 1
|
||||
)
|
||||
num_decodes = getattr(forward_batch, "_original_batch_size", None)
|
||||
if num_decodes is None:
|
||||
num_decodes = len(forward_batch.seq_lens)
|
||||
return cls.prepare_decode(
|
||||
forward_metadata,
|
||||
forward_batch.seq_lens,
|
||||
is_target_verify=forward_batch.forward_mode.is_target_verify(),
|
||||
draft_token_num=draft_token_num,
|
||||
num_decodes=num_decodes,
|
||||
)
|
||||
extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
if extend_seq_lens_cpu is None:
|
||||
@@ -255,10 +250,7 @@ class Mamba2Metadata(ForwardMetadata):
|
||||
num_prefill_tokens = int(sum(extend_seq_lens_cpu))
|
||||
else:
|
||||
num_prefill_tokens = int(forward_batch.extend_num_tokens)
|
||||
batch_size = getattr(forward_batch, "_original_batch_size", None)
|
||||
if batch_size is None:
|
||||
batch_size = len(forward_batch.seq_lens)
|
||||
num_decodes = max(0, batch_size - num_prefills)
|
||||
num_decodes = len(forward_batch.seq_lens) - num_prefills
|
||||
context_lens_tensor = forward_batch.extend_prefix_lens
|
||||
assert context_lens_tensor is not None
|
||||
has_initial_states = context_lens_tensor > 0
|
||||
|
||||
@@ -160,6 +160,11 @@ def _linear_attention_with_output_impl(
|
||||
real_num_tokens = min(
|
||||
forward_batch.global_num_token_non_padded_cpu, mixed_qkv.shape[0]
|
||||
)
|
||||
if real_num_tokens == 0:
|
||||
# A fully masked batch (an idle DP rank) needs no attention or state
|
||||
# update, and GDN prefill kernels reject an empty varlen batch.
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
original_out_cache_loc = forward_batch.out_cache_loc
|
||||
# Keep the original ForwardBatch object and only narrow cache locations for
|
||||
|
||||
@@ -1591,9 +1591,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
or self.forward_mode.is_draft_extend_v2()
|
||||
or self.forward_mode.is_idle()
|
||||
):
|
||||
# Mamba-hybrid families need the fabricated-row idle conversion
|
||||
# below; this includes their MTP draft workers, whose mamba-less
|
||||
# "*E" pattern makes mambaish_config return None.
|
||||
# TARGET_VERIFY counts as extend; hybrid-SSM verify batches keep their
|
||||
# verify layout. MTP draft workers ("*E", no mamba layer) count as hybrid.
|
||||
hybrid_ssm = mambaish_config(model_runner.model_config) is not None or (
|
||||
model_runner.is_draft_worker
|
||||
and getattr(
|
||||
@@ -1617,16 +1616,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self._original_forward_mode = self.forward_mode
|
||||
self.forward_mode = ForwardMode.EXTEND
|
||||
# Fabricate a single dummy request covering num_tokens for an
|
||||
# empty (idle) rank. Hybrid-SSM families always take this path;
|
||||
# non-hybrid ranks reach it once MAX_LEN is forced for the
|
||||
# prefill breakable CUDA graph (idle + prefill), which needs
|
||||
# empty (idle) rank. Ranks reach this once MAX_LEN is forced for
|
||||
# the prefill breakable CUDA graph (idle + prefill), which needs
|
||||
# every DP rank to run the same captured shape. The `else`
|
||||
# branch handles decode rows padded to a 1-token extend.
|
||||
if hybrid_ssm or self.seq_lens.shape[0] == 0:
|
||||
if self.seq_lens.shape[0] == 0:
|
||||
dev = self.seq_lens.device
|
||||
assert self.seq_lens.shape[0] == 0, (
|
||||
"extend-idle conversion expects an empty rank"
|
||||
)
|
||||
self.extend_num_tokens = num_tokens
|
||||
self.extend_seq_lens = torch.tensor(
|
||||
[num_tokens], dtype=torch.int32, device=dev
|
||||
@@ -1655,15 +1650,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self.extend_seq_lens_cpu = [int(num_tokens)]
|
||||
self.extend_logprob_start_lens_cpu = [0]
|
||||
bs = self.batch_size = 1
|
||||
# Keep idle non-hybrid fabricated rows masked by default.
|
||||
# Hybrid-SSM needs the real count for its state update.
|
||||
mask_dummy_tokens = (
|
||||
not hybrid_ssm and self._original_forward_mode.is_idle()
|
||||
)
|
||||
# Bump the GLOBAL scalar; the LOCAL count is derived from it
|
||||
# downstream. (global_num_token_non_padded is None unless
|
||||
# Keep idle fabricated rows masked. Bump the GLOBAL scalar;
|
||||
# the LOCAL count is derived from it downstream.
|
||||
# (global_num_token_non_padded is None unless
|
||||
# moe_ep_size > 1.)
|
||||
if mask_dummy_tokens:
|
||||
if self._original_forward_mode.is_idle():
|
||||
if self.global_num_token_non_padded is not None:
|
||||
self.global_num_token_non_padded.fill_(0)
|
||||
self.global_num_token_non_padded_cpu = 0
|
||||
|
||||
@@ -81,11 +81,9 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
replace_substrings,
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import (
|
||||
get_real_num_tokens,
|
||||
input_norm_maybe_fuse_allreduce,
|
||||
feeds_mlp_layer,
|
||||
is_attn_layer,
|
||||
make_layer_communicator,
|
||||
pad_to_original_num_tokens,
|
||||
)
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
@@ -383,7 +381,24 @@ class NemotronHMoE(nn.Module):
|
||||
|
||||
|
||||
class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
"""Shared forward for the dense-MLP / MoE decoder layers."""
|
||||
"""MLP half of a decoder layer. A preceding Mamba / attention layer hands over
|
||||
its unreduced output, as between the two halves of a standard decoder layer."""
|
||||
|
||||
def _init_layer_communicator(
|
||||
self, config: NemotronHConfig, layer_idx: int, *, is_sparse: bool
|
||||
) -> None:
|
||||
pattern = config.hybrid_override_pattern
|
||||
self.input_is_reduced = layer_idx == 0 or not is_attn_layer(
|
||||
pattern[layer_idx - 1]
|
||||
)
|
||||
self.feeds_mlp_layer = feeds_mlp_layer(pattern, layer_idx)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=False,
|
||||
allow_reduce_scatter=True,
|
||||
is_sparse=is_sparse,
|
||||
is_last_layer=layer_idx == len(pattern) - 1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -392,44 +407,36 @@ class NemotronHMLPLikeDecoderLayer(nn.Module):
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if is_dp_attention_enabled():
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = input_norm_maybe_fuse_allreduce(
|
||||
self.norm, hidden_states, residual
|
||||
if self.input_is_reduced:
|
||||
# Fold the reduced input into the residual, leaving prepare_mlp a
|
||||
# zero update to reduce.
|
||||
residual = hidden_states if residual is None else hidden_states + residual
|
||||
hidden_states = torch.zeros_like(hidden_states)
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
# prepare_mlp ignores a deferred reduction, so only defer into a
|
||||
# Mamba / attention layer.
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
not self.feeds_mlp_layer
|
||||
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=fuse_mlp_allreduce,
|
||||
mlp_reduce_scatter=mlp_reduce_scatter,
|
||||
):
|
||||
hidden_states = self.mixer.forward(hidden_states)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
else:
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -464,12 +471,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=False,
|
||||
allow_reduce_scatter=True,
|
||||
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
|
||||
)
|
||||
self._init_layer_communicator(config, layer_idx, is_sparse=False)
|
||||
|
||||
|
||||
class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
@@ -492,34 +494,50 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=False,
|
||||
allow_reduce_scatter=True,
|
||||
is_sparse=True,
|
||||
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
|
||||
)
|
||||
self._init_layer_communicator(config, layer_idx, is_sparse=True)
|
||||
|
||||
|
||||
class NemotronHAttnLikeDecoderLayer(nn.Module):
|
||||
"""Shared DP-attention input prep for the Mamba / full-attention layers."""
|
||||
"""Attention half of a decoder layer. Before an MLP layer the mixer leaves its
|
||||
output unreduced for prepare_mlp; otherwise the mixer reduces it itself."""
|
||||
|
||||
def _set_prev_layer_is_attn(self, config: NemotronHConfig, layer_idx: int) -> None:
|
||||
self.prev_layer_is_attn = layer_idx > 0 and is_attn_layer(
|
||||
config.hybrid_override_pattern[layer_idx - 1]
|
||||
def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int):
|
||||
self.feeds_mlp_layer = feeds_mlp_layer(
|
||||
config.hybrid_override_pattern, layer_idx
|
||||
)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=True,
|
||||
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
|
||||
)
|
||||
|
||||
def _dp_attn_input(
|
||||
def forward(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if self.prev_layer_is_attn and residual is not None:
|
||||
hidden_states = attn_tp_all_reduce(hidden_states)
|
||||
return self.layer_communicator.prepare_attn(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
hidden_states, residual = self.layer_communicator.prepare_attn(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return hidden_states, residual
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
not self.feeds_mlp_layer
|
||||
and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
skip_reduce = self.feeds_mlp_layer or fuse_mlp_allreduce
|
||||
with get_forward().scoped(fuse_mlp_allreduce=skip_reduce):
|
||||
hidden_states = self._forward_mixer(
|
||||
hidden_states, forward_batch, skip_reduce
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
@@ -542,16 +560,12 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
rms_norm_eps=config.layer_norm_epsilon,
|
||||
activation=config.mamba_hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
prefix=f"{prefix}.mixer",
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=True,
|
||||
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
|
||||
)
|
||||
self._set_prev_layer_is_attn(config, layer_idx)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
|
||||
def _forward_mamba(
|
||||
self,
|
||||
@@ -559,15 +573,10 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
"""Core Mamba forward logic, called directly or via split op."""
|
||||
original_num_tokens = hidden_states.shape[0]
|
||||
if forward_batch.forward_mode.is_extend():
|
||||
real_num_tokens = get_real_num_tokens(hidden_states, forward_batch)
|
||||
if real_num_tokens < original_num_tokens:
|
||||
hidden_states = hidden_states[:real_num_tokens]
|
||||
attn_backend = get_attn_backend()
|
||||
assert isinstance(attn_backend, HybridLinearAttnBackend)
|
||||
assert isinstance(attn_backend.linear_attn_backend, Mamba2AttnBackend)
|
||||
output = attn_backend.linear_attn_backend.forward(
|
||||
return attn_backend.linear_attn_backend.forward(
|
||||
mixer=self.mixer,
|
||||
layer_id=self.layer_id,
|
||||
hidden_states=hidden_states,
|
||||
@@ -575,61 +584,26 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
forward_batch=forward_batch,
|
||||
use_triton_causal_conv=True,
|
||||
)
|
||||
return pad_to_original_num_tokens(output, original_num_tokens)
|
||||
|
||||
def forward(
|
||||
def _forward_mixer(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if is_dp_attention_enabled():
|
||||
hidden_states, residual = self._dp_attn_input(
|
||||
hidden_states, residual, forward_batch
|
||||
skip_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
if is_in_breakable_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
breakable_nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, skip_reduce
|
||||
)
|
||||
if get_real_num_tokens(hidden_states, forward_batch) == 0:
|
||||
return torch.zeros_like(hidden_states), residual
|
||||
|
||||
if is_in_breakable_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
breakable_nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, False
|
||||
)
|
||||
elif is_in_tc_piecewise_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
nemotron_mamba2_with_output(hidden_states, output, self.layer_id, False)
|
||||
else:
|
||||
output = self._forward_mamba(hidden_states, forward_batch)
|
||||
return output, residual
|
||||
|
||||
hidden_states, residual = input_norm_maybe_fuse_allreduce(
|
||||
self.norm, hidden_states, residual
|
||||
)
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
return output
|
||||
if is_in_tc_piecewise_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, skip_reduce
|
||||
)
|
||||
)
|
||||
|
||||
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
|
||||
if is_in_breakable_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
breakable_nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, fuse_mlp_allreduce
|
||||
)
|
||||
elif is_in_tc_piecewise_cuda_graph():
|
||||
output = torch.empty_like(hidden_states)
|
||||
nemotron_mamba2_with_output(
|
||||
hidden_states, output, self.layer_id, fuse_mlp_allreduce
|
||||
)
|
||||
else:
|
||||
output = self._forward_mamba(hidden_states, forward_batch)
|
||||
|
||||
if fuse_mlp_allreduce:
|
||||
output._sglang_needs_allreduce_fusion = True
|
||||
return output, residual
|
||||
return output
|
||||
return self._forward_mamba(hidden_states, forward_batch)
|
||||
|
||||
|
||||
class NemotronHAttention(nn.Module):
|
||||
@@ -683,7 +657,7 @@ class NemotronHAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
reduce_results=not is_dp_attention_enabled(),
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
@@ -703,38 +677,9 @@ class NemotronHAttention(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
if not is_dp_attention_enabled():
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
attn_output = self.attn.forward(q, k, v, forward_batch)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
padded_shape = hidden_states.shape[0]
|
||||
real_tokens = get_real_num_tokens(hidden_states, forward_batch)
|
||||
has_padding = real_tokens < padded_shape
|
||||
keep_q_padded = (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_idle()
|
||||
or forward_batch._original_forward_mode is not None
|
||||
)
|
||||
original_out_cache_loc = forward_batch.out_cache_loc
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
if has_padding and real_tokens > 0:
|
||||
k, v = k[:real_tokens], v[:real_tokens]
|
||||
if original_out_cache_loc is not None:
|
||||
forward_batch.out_cache_loc = original_out_cache_loc[:real_tokens]
|
||||
if not keep_q_padded:
|
||||
q = q[:real_tokens]
|
||||
attn_output = self.attn.forward(
|
||||
q, k, v, forward_batch, save_kv_cache=real_tokens > 0
|
||||
)
|
||||
forward_batch.out_cache_loc = original_out_cache_loc
|
||||
|
||||
attn_output = pad_to_original_num_tokens(attn_output, padded_shape)
|
||||
attn_output = self.attn.forward(q, k, v, forward_batch)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
@@ -758,48 +703,18 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer):
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
self.layer_communicator = make_layer_communicator(
|
||||
self.norm,
|
||||
for_attn=True,
|
||||
is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1,
|
||||
)
|
||||
self._set_prev_layer_is_attn(config, layer_idx)
|
||||
self._init_layer_communicator(config, layer_idx)
|
||||
|
||||
def forward(
|
||||
def _forward_mixer(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if is_dp_attention_enabled():
|
||||
hidden_states, residual = self._dp_attn_input(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
hidden_states = self.mixer.forward(
|
||||
hidden_states=hidden_states, forward_batch=forward_batch
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
hidden_states, residual = input_norm_maybe_fuse_allreduce(
|
||||
self.norm, hidden_states, residual
|
||||
skip_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
return self.mixer.forward(
|
||||
hidden_states=hidden_states, forward_batch=forward_batch
|
||||
)
|
||||
|
||||
fuse_mlp_allreduce = (
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer(
|
||||
forward_batch
|
||||
)
|
||||
)
|
||||
|
||||
with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce):
|
||||
hidden_states = self.mixer.forward(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
hidden_states._sglang_needs_allreduce_fusion = True
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
Layers = (
|
||||
NemotronHAttentionDecoderLayer,
|
||||
@@ -864,13 +779,14 @@ class NemotronHModel(nn.Module):
|
||||
self.layers_to_capture: set[int] = set()
|
||||
|
||||
def _capture_hidden_states(self, hidden_states, residual, boundary_idx):
|
||||
pattern = self.config.hybrid_override_pattern
|
||||
if (
|
||||
is_dp_attention_enabled()
|
||||
and residual is not None
|
||||
residual is not None
|
||||
and boundary_idx > 0
|
||||
and is_attn_layer(self.config.hybrid_override_pattern[boundary_idx - 1])
|
||||
and is_attn_layer(pattern[boundary_idx - 1])
|
||||
and feeds_mlp_layer(pattern, boundary_idx - 1)
|
||||
):
|
||||
# Reduce a copy so the next layer still receives a TP partial.
|
||||
# Reduce a copy so the MLP layer still receives a TP partial.
|
||||
hidden_states = attn_tp_all_reduce(hidden_states.clone())
|
||||
# Later norms update the input residual in place.
|
||||
return hidden_states.clone() if residual is None else hidden_states + residual
|
||||
@@ -927,7 +843,10 @@ class NemotronHModel(nn.Module):
|
||||
aux_hidden_states.append(
|
||||
self._capture_hidden_states(hidden_states, residual, self.end_layer)
|
||||
)
|
||||
hidden_states, _ = self.norm_f(hidden_states, residual)
|
||||
if residual is None:
|
||||
hidden_states = self.norm_f(hidden_states)
|
||||
else:
|
||||
hidden_states, _ = self.norm_f(hidden_states, residual)
|
||||
if aux_hidden_states:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
"""DP-attention helpers for the Nemotron-H model."""
|
||||
"""Layer-communication helpers for the Nemotron-H model."""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA
|
||||
from sglang.srt.distributed import tensor_model_parallel_all_reduce
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
ScatterMode,
|
||||
apply_flashinfer_allreduce_fusion,
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
ATTN_LAYERS = (MAMBA, ATTENTION)
|
||||
|
||||
@@ -22,32 +18,9 @@ def is_attn_layer(layer_type: str) -> bool:
|
||||
return layer_type in ATTN_LAYERS
|
||||
|
||||
|
||||
def get_real_num_tokens(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> int:
|
||||
"""Number of real (non DP-padding) rows in ``hidden_states``."""
|
||||
real_tokens = hidden_states.shape[0]
|
||||
if forward_batch.global_num_token_non_padded_cpu is not None:
|
||||
real_tokens = min(
|
||||
real_tokens, int(forward_batch.global_num_token_non_padded_cpu)
|
||||
)
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and not forward_batch.forward_mode.is_mixed()
|
||||
and forward_batch.extend_seq_lens_cpu is not None
|
||||
):
|
||||
real_tokens = min(real_tokens, int(sum(forward_batch.extend_seq_lens_cpu)))
|
||||
return real_tokens
|
||||
|
||||
|
||||
def pad_to_original_num_tokens(
|
||||
output: torch.Tensor, original_num_tokens: int
|
||||
) -> torch.Tensor:
|
||||
if output.shape[0] == original_num_tokens:
|
||||
return output
|
||||
padded = output.new_empty((original_num_tokens, *output.shape[1:]))
|
||||
padded[: output.shape[0]] = output
|
||||
return padded
|
||||
def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool:
|
||||
next_idx = layer_idx + 1
|
||||
return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx])
|
||||
|
||||
|
||||
def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes:
|
||||
@@ -77,31 +50,9 @@ def make_layer_communicator(
|
||||
layer_scatter_modes=_build_layer_scatter_modes(is_sparse),
|
||||
input_layernorm=layer_norm if for_attn else nn.Identity(),
|
||||
post_attention_layernorm=nn.Identity() if for_attn else layer_norm,
|
||||
# With attention TP > 1, the default gather adds the residual to one
|
||||
# rank's partial in bf16 before the cross-rank sum.
|
||||
force_layernorm_before_dp_gather=True,
|
||||
allow_reduce_scatter=allow_reduce_scatter,
|
||||
is_last_layer=is_last_layer,
|
||||
)
|
||||
|
||||
|
||||
def input_norm_maybe_fuse_allreduce(
|
||||
norm: RMSNorm,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is not None and getattr(
|
||||
hidden_states, "_sglang_needs_allreduce_fusion", False
|
||||
):
|
||||
if apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) and hasattr(
|
||||
norm, "forward_with_allreduce_fusion"
|
||||
):
|
||||
return norm.forward_with_allreduce_fusion(
|
||||
hidden_states, residual, use_attn_tp_group=False
|
||||
)
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = norm(hidden_states)
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
Reference in New Issue
Block a user