From 701cf7e47e6815d49ce9aa4124e61d3db8e16f8a Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:40:33 -0700 Subject: [PATCH] [Refactor] Run Nemotron-H DP attention through the standard layer communicator (#40867) --- .../attention/hybrid_linear_attn_backend.py | 15 +- .../srt/layers/attention/mamba/mamba.py | 10 +- .../layers/attention/mamba/mamba2_metadata.py | 12 +- .../srt/layers/radix_linear_attention.py | 5 + .../srt/model_executor/forward_batch_info.py | 27 +- python/sglang/srt/models/nemotron_h.py | 297 +++++++----------- python/sglang/srt/models/nemotron_h_utils.py | 61 +--- .../test_linear_attention_masked_idle_rank.py | 54 ++++ .../test_mlp_sync_hybrid_target_verify.py | 96 ++++++ .../models/test_nemotron_h_aux_capture.py | 34 +- .../models/test_nemotron_h_mtp_reduction.py | 8 +- 11 files changed, 318 insertions(+), 301 deletions(-) create mode 100644 test/registered/unit/layers/test_linear_attention_masked_idle_rank.py create mode 100644 test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 6249cb6b5b..e04f2fe354 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -157,11 +157,11 @@ class MambaAttnBackendBase(AttentionBackend): logical_num_tokens = None track_mask_indices = None + # Padded rows carry req-pool row 0, which maps to the reserved mamba + # padding slot 0 in both the static and the unified pool. mamba_cache_indices = self.req_to_token_pool.get_mamba_indices( forward_batch.req_pool_indices ) - # Translate virtual->physical BEFORE the padding sentinel below, so the - # gather reads only real ids; padded rows are then poisoned to -1 (skipped). mamba_cache_indices = self._translate_mamba_indices(mamba_cache_indices) if forward_batch.mamba_track_indices is not None: forward_batch.mamba_track_indices = self._translate_mamba_indices( @@ -182,11 +182,6 @@ class MambaAttnBackendBase(AttentionBackend): forward_batch.mamba_track_mask is not None and forward_batch.mamba_track_mask.any() ) - _real_bs = forward_batch._original_batch_size - if _real_bs is not None and _real_bs < mamba_cache_indices.shape[0]: - mamba_cache_indices = mamba_cache_indices.clone() - mamba_cache_indices[_real_bs:] = -1 - replayssm_write_pos = None replayssm_force_flush = None if forward_batch.forward_mode.is_decode_or_idle(): @@ -209,8 +204,8 @@ class MambaAttnBackendBase(AttentionBackend): ) if write_pos_buf is not None: slots = mamba_cache_indices.to(torch.long) - # Padded rows carry slot == -1; clamp the gather in-bounds (kernel - # zeroes padded rows via state_idx < 0). + # Padded rows point at the reserved padding slot 0; the clamp keeps + # a -1 sentinel in bounds (the kernel zeroes rows with state_idx < 0). safe_slots = slots.clamp(min=0) replayssm_write_pos = write_pos_buf[safe_slots].clone() L = mamba_pool.linear_replayssm_cache_len @@ -227,7 +222,7 @@ class MambaAttnBackendBase(AttentionBackend): device=self.device, dtype=torch.int32 ) # Advance only valid slots, scattered over unique slots (dup-index - # race; padded rows clamp to 0); a forced flush -> next write_pos 0. + # race; padded rows share slot 0); a forced flush -> next write_pos 0. valid_mask = slots >= 0 valid_slots = slots[valid_mask] if valid_slots.numel() > 0: diff --git a/python/sglang/srt/layers/attention/mamba/mamba.py b/python/sglang/srt/layers/attention/mamba/mamba.py index 0778c3ab68..a644858c5a 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba.py +++ b/python/sglang/srt/layers/attention/mamba/mamba.py @@ -211,6 +211,7 @@ class MambaMixer2(torch.nn.Module): activation: str = "silu", use_rms_norm: bool = True, quant_config: Optional[QuantizationConfig] = None, + reduce_results: Optional[bool] = None, prefix: str = "", ): super().__init__() @@ -421,6 +422,9 @@ class MambaMixer2(torch.nn.Module): set_weight_attrs(self.A, {"weight_loader": a_weight_loader}) set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)}) + # By default a layer communicator reduces the output under DP attention. + if reduce_results is None: + reduce_results = not is_dp_attention_enabled() self.out_proj = RowParallelLinear( intermediate_size, hidden_size, @@ -429,7 +433,8 @@ class MambaMixer2(torch.nn.Module): quant_config=quant_config, tp_rank=self.tp_rank, tp_size=self.tp_size, - reduce_results=not is_dp_attention_enabled(), + reduce_results=reduce_results, + use_dp_attention_reduce=reduce_results and is_dp_attention_enabled(), prefix=f"{prefix}.out_proj", ) @@ -545,6 +550,9 @@ class MambaMixer2(torch.nn.Module): dtype=hidden_states.dtype, device=hidden_states.device, ) + # Rows past the batch's tokens are DP padding; keep them finite. + if num_actual_tokens < preallocated_ssm_out.shape[0]: + preallocated_ssm_out[num_actual_tokens:].zero_() preallocated_ssm_out_active = preallocated_ssm_out[:num_actual_tokens] preallocated_ssm_out_p, preallocated_ssm_out_d = torch.split( preallocated_ssm_out_active, diff --git a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py index c0cba1d8b8..b2843a536e 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py +++ b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py @@ -196,7 +196,6 @@ class Mamba2Metadata(ForwardMetadata): *, is_target_verify: bool, draft_token_num: int, - num_decodes: Optional[int] = None, ) -> "Mamba2Metadata": """This path is run during CUDA graph capture, i.e. decode only, so `num_prefills` is 0""" return Mamba2Metadata( @@ -215,7 +214,7 @@ class Mamba2Metadata(ForwardMetadata): track_ssm_end_locs=forward_metadata.track_ssm_end_locs, track_ssm_recompute_dst=forward_metadata.track_ssm_recompute_dst, has_mamba_track_mask=forward_metadata.has_mamba_track_mask, - num_decodes=len(seq_lens) if num_decodes is None else num_decodes, + num_decodes=len(seq_lens), num_prefills=0, num_prefill_tokens=0, is_target_verify=is_target_verify, @@ -236,15 +235,11 @@ class Mamba2Metadata(ForwardMetadata): if forward_batch.spec_info is not None else 1 ) - num_decodes = getattr(forward_batch, "_original_batch_size", None) - if num_decodes is None: - num_decodes = len(forward_batch.seq_lens) return cls.prepare_decode( forward_metadata, forward_batch.seq_lens, is_target_verify=forward_batch.forward_mode.is_target_verify(), draft_token_num=draft_token_num, - num_decodes=num_decodes, ) extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu if extend_seq_lens_cpu is None: @@ -255,10 +250,7 @@ class Mamba2Metadata(ForwardMetadata): num_prefill_tokens = int(sum(extend_seq_lens_cpu)) else: num_prefill_tokens = int(forward_batch.extend_num_tokens) - batch_size = getattr(forward_batch, "_original_batch_size", None) - if batch_size is None: - batch_size = len(forward_batch.seq_lens) - num_decodes = max(0, batch_size - num_prefills) + num_decodes = len(forward_batch.seq_lens) - num_prefills context_lens_tensor = forward_batch.extend_prefix_lens assert context_lens_tensor is not None has_initial_states = context_lens_tensor > 0 diff --git a/python/sglang/srt/layers/radix_linear_attention.py b/python/sglang/srt/layers/radix_linear_attention.py index 6c6218820a..39201effbe 100644 --- a/python/sglang/srt/layers/radix_linear_attention.py +++ b/python/sglang/srt/layers/radix_linear_attention.py @@ -160,6 +160,11 @@ def _linear_attention_with_output_impl( real_num_tokens = min( forward_batch.global_num_token_non_padded_cpu, mixed_qkv.shape[0] ) + if real_num_tokens == 0: + # A fully masked batch (an idle DP rank) needs no attention or state + # update, and GDN prefill kernels reject an empty varlen batch. + output.zero_() + return original_out_cache_loc = forward_batch.out_cache_loc # Keep the original ForwardBatch object and only narrow cache locations for diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 028c4fcb96..8ef752766d 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1591,9 +1591,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): or self.forward_mode.is_draft_extend_v2() or self.forward_mode.is_idle() ): - # Mamba-hybrid families need the fabricated-row idle conversion - # below; this includes their MTP draft workers, whose mamba-less - # "*E" pattern makes mambaish_config return None. + # TARGET_VERIFY counts as extend; hybrid-SSM verify batches keep their + # verify layout. MTP draft workers ("*E", no mamba layer) count as hybrid. hybrid_ssm = mambaish_config(model_runner.model_config) is not None or ( model_runner.is_draft_worker and getattr( @@ -1617,16 +1616,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self._original_forward_mode = self.forward_mode self.forward_mode = ForwardMode.EXTEND # Fabricate a single dummy request covering num_tokens for an - # empty (idle) rank. Hybrid-SSM families always take this path; - # non-hybrid ranks reach it once MAX_LEN is forced for the - # prefill breakable CUDA graph (idle + prefill), which needs + # empty (idle) rank. Ranks reach this once MAX_LEN is forced for + # the prefill breakable CUDA graph (idle + prefill), which needs # every DP rank to run the same captured shape. The `else` # branch handles decode rows padded to a 1-token extend. - if hybrid_ssm or self.seq_lens.shape[0] == 0: + if self.seq_lens.shape[0] == 0: dev = self.seq_lens.device - assert self.seq_lens.shape[0] == 0, ( - "extend-idle conversion expects an empty rank" - ) self.extend_num_tokens = num_tokens self.extend_seq_lens = torch.tensor( [num_tokens], dtype=torch.int32, device=dev @@ -1655,15 +1650,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self.extend_seq_lens_cpu = [int(num_tokens)] self.extend_logprob_start_lens_cpu = [0] bs = self.batch_size = 1 - # Keep idle non-hybrid fabricated rows masked by default. - # Hybrid-SSM needs the real count for its state update. - mask_dummy_tokens = ( - not hybrid_ssm and self._original_forward_mode.is_idle() - ) - # Bump the GLOBAL scalar; the LOCAL count is derived from it - # downstream. (global_num_token_non_padded is None unless + # Keep idle fabricated rows masked. Bump the GLOBAL scalar; + # the LOCAL count is derived from it downstream. + # (global_num_token_non_padded is None unless # moe_ep_size > 1.) - if mask_dummy_tokens: + if self._original_forward_mode.is_idle(): if self.global_num_token_non_padded is not None: self.global_num_token_non_padded.fill_(0) self.global_num_token_non_padded_cpu = 0 diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 8788dd7fdc..8f1186b413 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -81,11 +81,9 @@ from sglang.srt.model_loader.weight_utils import ( replace_substrings, ) from sglang.srt.models.nemotron_h_utils import ( - get_real_num_tokens, - input_norm_maybe_fuse_allreduce, + feeds_mlp_layer, is_attn_layer, make_layer_communicator, - pad_to_original_num_tokens, ) from sglang.srt.models.utils import WeightsMapper from sglang.srt.runtime_context import get_exec, get_forward, get_parallel @@ -383,7 +381,24 @@ class NemotronHMoE(nn.Module): class NemotronHMLPLikeDecoderLayer(nn.Module): - """Shared forward for the dense-MLP / MoE decoder layers.""" + """MLP half of a decoder layer. A preceding Mamba / attention layer hands over + its unreduced output, as between the two halves of a standard decoder layer.""" + + def _init_layer_communicator( + self, config: NemotronHConfig, layer_idx: int, *, is_sparse: bool + ) -> None: + pattern = config.hybrid_override_pattern + self.input_is_reduced = layer_idx == 0 or not is_attn_layer( + pattern[layer_idx - 1] + ) + self.feeds_mlp_layer = feeds_mlp_layer(pattern, layer_idx) + self.layer_communicator = make_layer_communicator( + self.norm, + for_attn=False, + allow_reduce_scatter=True, + is_sparse=is_sparse, + is_last_layer=layer_idx == len(pattern) - 1, + ) def forward( self, @@ -392,44 +407,36 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): residual: torch.Tensor | None, forward_batch: ForwardBatch, ) -> tuple[torch.Tensor, torch.Tensor]: - if is_dp_attention_enabled(): - hidden_states, residual = self.layer_communicator.prepare_mlp( - hidden_states, residual, forward_batch - ) - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( - forward_batch - ) - ) - with get_forward().scoped( - fuse_mlp_allreduce=fuse_mlp_allreduce, - mlp_reduce_scatter=mlp_reduce_scatter, - ): - hidden_states = self.mixer.forward(hidden_states) - if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True - else: - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) - return hidden_states, residual - - hidden_states, residual = input_norm_maybe_fuse_allreduce( - self.norm, hidden_states, residual + if self.input_is_reduced: + # Fold the reduced input into the residual, leaving prepare_mlp a + # zero update to reduce. + residual = hidden_states if residual is None else hidden_states + residual + hidden_states = torch.zeros_like(hidden_states) + hidden_states, residual = self.layer_communicator.prepare_mlp( + hidden_states, residual, forward_batch ) - + mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( + forward_batch + ) + # prepare_mlp ignores a deferred reduction, so only defer into a + # Mamba / attention layer. fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( + not self.feeds_mlp_layer + and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( forward_batch ) ) - with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce): + with get_forward().scoped( + fuse_mlp_allreduce=fuse_mlp_allreduce, + mlp_reduce_scatter=mlp_reduce_scatter, + ): hidden_states = self.mixer.forward(hidden_states) if fuse_mlp_allreduce: hidden_states._sglang_needs_allreduce_fusion = True + else: + hidden_states, residual = self.layer_communicator.postprocess_layer( + hidden_states, residual, forward_batch + ) return hidden_states, residual @@ -464,12 +471,7 @@ class NemotronHMLPDecoderLayer(NemotronHMLPLikeDecoderLayer): ) self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - self.layer_communicator = make_layer_communicator( - self.norm, - for_attn=False, - allow_reduce_scatter=True, - is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1, - ) + self._init_layer_communicator(config, layer_idx, is_sparse=False) class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer): @@ -492,34 +494,50 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer): ) self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - self.layer_communicator = make_layer_communicator( - self.norm, - for_attn=False, - allow_reduce_scatter=True, - is_sparse=True, - is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1, - ) + self._init_layer_communicator(config, layer_idx, is_sparse=True) class NemotronHAttnLikeDecoderLayer(nn.Module): - """Shared DP-attention input prep for the Mamba / full-attention layers.""" + """Attention half of a decoder layer. Before an MLP layer the mixer leaves its + output unreduced for prepare_mlp; otherwise the mixer reduces it itself.""" - def _set_prev_layer_is_attn(self, config: NemotronHConfig, layer_idx: int) -> None: - self.prev_layer_is_attn = layer_idx > 0 and is_attn_layer( - config.hybrid_override_pattern[layer_idx - 1] + def _init_layer_communicator(self, config: NemotronHConfig, layer_idx: int): + self.feeds_mlp_layer = feeds_mlp_layer( + config.hybrid_override_pattern, layer_idx + ) + self.layer_communicator = make_layer_communicator( + self.norm, + for_attn=True, + is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1, ) - def _dp_attn_input( + def forward( self, + *, hidden_states: torch.Tensor, residual: torch.Tensor | None, forward_batch: ForwardBatch, - ) -> tuple[torch.Tensor, torch.Tensor | None]: - if self.prev_layer_is_attn and residual is not None: - hidden_states = attn_tp_all_reduce(hidden_states) - return self.layer_communicator.prepare_attn( + ) -> tuple[torch.Tensor, torch.Tensor]: + hidden_states, residual = self.layer_communicator.prepare_attn( hidden_states, residual, forward_batch ) + if forward_batch.forward_mode.is_idle(): + return hidden_states, residual + + fuse_mlp_allreduce = ( + not self.feeds_mlp_layer + and self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( + forward_batch + ) + ) + skip_reduce = self.feeds_mlp_layer or fuse_mlp_allreduce + with get_forward().scoped(fuse_mlp_allreduce=skip_reduce): + hidden_states = self._forward_mixer( + hidden_states, forward_batch, skip_reduce + ) + if fuse_mlp_allreduce: + hidden_states._sglang_needs_allreduce_fusion = True + return hidden_states, residual class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): @@ -542,16 +560,12 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): rms_norm_eps=config.layer_norm_epsilon, activation=config.mamba_hidden_act, quant_config=quant_config, + reduce_results=True, prefix=f"{prefix}.mixer", ) self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - self.layer_communicator = make_layer_communicator( - self.norm, - for_attn=True, - is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1, - ) - self._set_prev_layer_is_attn(config, layer_idx) + self._init_layer_communicator(config, layer_idx) def _forward_mamba( self, @@ -559,15 +573,10 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): forward_batch: ForwardBatch, ) -> torch.Tensor: """Core Mamba forward logic, called directly or via split op.""" - original_num_tokens = hidden_states.shape[0] - if forward_batch.forward_mode.is_extend(): - real_num_tokens = get_real_num_tokens(hidden_states, forward_batch) - if real_num_tokens < original_num_tokens: - hidden_states = hidden_states[:real_num_tokens] attn_backend = get_attn_backend() assert isinstance(attn_backend, HybridLinearAttnBackend) assert isinstance(attn_backend.linear_attn_backend, Mamba2AttnBackend) - output = attn_backend.linear_attn_backend.forward( + return attn_backend.linear_attn_backend.forward( mixer=self.mixer, layer_id=self.layer_id, hidden_states=hidden_states, @@ -575,61 +584,26 @@ class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): forward_batch=forward_batch, use_triton_causal_conv=True, ) - return pad_to_original_num_tokens(output, original_num_tokens) - def forward( + def _forward_mixer( self, - *, hidden_states: torch.Tensor, - residual: torch.Tensor | None, forward_batch: ForwardBatch, - ) -> tuple[torch.Tensor, torch.Tensor]: - if is_dp_attention_enabled(): - hidden_states, residual = self._dp_attn_input( - hidden_states, residual, forward_batch + skip_reduce: bool, + ) -> torch.Tensor: + if is_in_breakable_cuda_graph(): + output = torch.empty_like(hidden_states) + breakable_nemotron_mamba2_with_output( + hidden_states, output, self.layer_id, skip_reduce ) - if get_real_num_tokens(hidden_states, forward_batch) == 0: - return torch.zeros_like(hidden_states), residual - - if is_in_breakable_cuda_graph(): - output = torch.empty_like(hidden_states) - breakable_nemotron_mamba2_with_output( - hidden_states, output, self.layer_id, False - ) - elif is_in_tc_piecewise_cuda_graph(): - output = torch.empty_like(hidden_states) - nemotron_mamba2_with_output(hidden_states, output, self.layer_id, False) - else: - output = self._forward_mamba(hidden_states, forward_batch) - return output, residual - - hidden_states, residual = input_norm_maybe_fuse_allreduce( - self.norm, hidden_states, residual - ) - - fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( - forward_batch + return output + if is_in_tc_piecewise_cuda_graph(): + output = torch.empty_like(hidden_states) + nemotron_mamba2_with_output( + hidden_states, output, self.layer_id, skip_reduce ) - ) - - with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce): - if is_in_breakable_cuda_graph(): - output = torch.empty_like(hidden_states) - breakable_nemotron_mamba2_with_output( - hidden_states, output, self.layer_id, fuse_mlp_allreduce - ) - elif is_in_tc_piecewise_cuda_graph(): - output = torch.empty_like(hidden_states) - nemotron_mamba2_with_output( - hidden_states, output, self.layer_id, fuse_mlp_allreduce - ) - else: - output = self._forward_mamba(hidden_states, forward_batch) - - if fuse_mlp_allreduce: - output._sglang_needs_allreduce_fusion = True - return output, residual + return output + return self._forward_mamba(hidden_states, forward_batch) class NemotronHAttention(nn.Module): @@ -683,7 +657,7 @@ class NemotronHAttention(nn.Module): quant_config=quant_config, tp_rank=tp_rank, tp_size=tp_size, - reduce_results=not is_dp_attention_enabled(), + use_dp_attention_reduce=is_dp_attention_enabled(), prefix=f"{prefix}.o_proj", ) @@ -703,38 +677,9 @@ class NemotronHAttention(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - if not is_dp_attention_enabled(): - qkv, _ = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - attn_output = self.attn.forward(q, k, v, forward_batch) - output, _ = self.o_proj(attn_output) - return output - - padded_shape = hidden_states.shape[0] - real_tokens = get_real_num_tokens(hidden_states, forward_batch) - has_padding = real_tokens < padded_shape - keep_q_padded = ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_target_verify() - or forward_batch.forward_mode.is_idle() - or forward_batch._original_forward_mode is not None - ) - original_out_cache_loc = forward_batch.out_cache_loc - qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - if has_padding and real_tokens > 0: - k, v = k[:real_tokens], v[:real_tokens] - if original_out_cache_loc is not None: - forward_batch.out_cache_loc = original_out_cache_loc[:real_tokens] - if not keep_q_padded: - q = q[:real_tokens] - attn_output = self.attn.forward( - q, k, v, forward_batch, save_kv_cache=real_tokens > 0 - ) - forward_batch.out_cache_loc = original_out_cache_loc - - attn_output = pad_to_original_num_tokens(attn_output, padded_shape) + attn_output = self.attn.forward(q, k, v, forward_batch) output, _ = self.o_proj(attn_output) return output @@ -758,48 +703,18 @@ class NemotronHAttentionDecoderLayer(NemotronHAttnLikeDecoderLayer): ) self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) - self.layer_communicator = make_layer_communicator( - self.norm, - for_attn=True, - is_last_layer=layer_idx == len(config.hybrid_override_pattern) - 1, - ) - self._set_prev_layer_is_attn(config, layer_idx) + self._init_layer_communicator(config, layer_idx) - def forward( + def _forward_mixer( self, - *, hidden_states: torch.Tensor, - residual: torch.Tensor | None, forward_batch: ForwardBatch, - ) -> tuple[torch.Tensor, torch.Tensor]: - if is_dp_attention_enabled(): - hidden_states, residual = self._dp_attn_input( - hidden_states, residual, forward_batch - ) - hidden_states = self.mixer.forward( - hidden_states=hidden_states, forward_batch=forward_batch - ) - return hidden_states, residual - - hidden_states, residual = input_norm_maybe_fuse_allreduce( - self.norm, hidden_states, residual + skip_reduce: bool, + ) -> torch.Tensor: + return self.mixer.forward( + hidden_states=hidden_states, forward_batch=forward_batch ) - fuse_mlp_allreduce = ( - self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( - forward_batch - ) - ) - - with get_forward().scoped(fuse_mlp_allreduce=fuse_mlp_allreduce): - hidden_states = self.mixer.forward( - hidden_states=hidden_states, - forward_batch=forward_batch, - ) - if fuse_mlp_allreduce: - hidden_states._sglang_needs_allreduce_fusion = True - return hidden_states, residual - Layers = ( NemotronHAttentionDecoderLayer, @@ -864,13 +779,14 @@ class NemotronHModel(nn.Module): self.layers_to_capture: set[int] = set() def _capture_hidden_states(self, hidden_states, residual, boundary_idx): + pattern = self.config.hybrid_override_pattern if ( - is_dp_attention_enabled() - and residual is not None + residual is not None and boundary_idx > 0 - and is_attn_layer(self.config.hybrid_override_pattern[boundary_idx - 1]) + and is_attn_layer(pattern[boundary_idx - 1]) + and feeds_mlp_layer(pattern, boundary_idx - 1) ): - # Reduce a copy so the next layer still receives a TP partial. + # Reduce a copy so the MLP layer still receives a TP partial. hidden_states = attn_tp_all_reduce(hidden_states.clone()) # Later norms update the input residual in place. return hidden_states.clone() if residual is None else hidden_states + residual @@ -927,7 +843,10 @@ class NemotronHModel(nn.Module): aux_hidden_states.append( self._capture_hidden_states(hidden_states, residual, self.end_layer) ) - hidden_states, _ = self.norm_f(hidden_states, residual) + if residual is None: + hidden_states = self.norm_f(hidden_states) + else: + hidden_states, _ = self.norm_f(hidden_states, residual) if aux_hidden_states: return hidden_states, aux_hidden_states return hidden_states diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index 3e3240a978..ecace8e27a 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -1,19 +1,15 @@ -"""DP-attention helpers for the Nemotron-H model.""" +"""Layer-communication helpers for the Nemotron-H model.""" -import torch from torch import nn from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA -from sglang.srt.distributed import tensor_model_parallel_all_reduce from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, - apply_flashinfer_allreduce_fusion, ) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.moe.utils import get_moe_a2a_backend -from sglang.srt.model_executor.forward_batch_info import ForwardBatch ATTN_LAYERS = (MAMBA, ATTENTION) @@ -22,32 +18,9 @@ def is_attn_layer(layer_type: str) -> bool: return layer_type in ATTN_LAYERS -def get_real_num_tokens( - hidden_states: torch.Tensor, forward_batch: ForwardBatch -) -> int: - """Number of real (non DP-padding) rows in ``hidden_states``.""" - real_tokens = hidden_states.shape[0] - if forward_batch.global_num_token_non_padded_cpu is not None: - real_tokens = min( - real_tokens, int(forward_batch.global_num_token_non_padded_cpu) - ) - if ( - forward_batch.forward_mode.is_extend() - and not forward_batch.forward_mode.is_mixed() - and forward_batch.extend_seq_lens_cpu is not None - ): - real_tokens = min(real_tokens, int(sum(forward_batch.extend_seq_lens_cpu))) - return real_tokens - - -def pad_to_original_num_tokens( - output: torch.Tensor, original_num_tokens: int -) -> torch.Tensor: - if output.shape[0] == original_num_tokens: - return output - padded = output.new_empty((original_num_tokens, *output.shape[1:])) - padded[: output.shape[0]] = output - return padded +def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool: + next_idx = layer_idx + 1 + return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx]) def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes: @@ -77,31 +50,9 @@ def make_layer_communicator( layer_scatter_modes=_build_layer_scatter_modes(is_sparse), input_layernorm=layer_norm if for_attn else nn.Identity(), post_attention_layernorm=nn.Identity() if for_attn else layer_norm, + # With attention TP > 1, the default gather adds the residual to one + # rank's partial in bf16 before the cross-rank sum. force_layernorm_before_dp_gather=True, allow_reduce_scatter=allow_reduce_scatter, is_last_layer=is_last_layer, ) - - -def input_norm_maybe_fuse_allreduce( - norm: RMSNorm, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, -) -> tuple[torch.Tensor, torch.Tensor]: - if residual is not None and getattr( - hidden_states, "_sglang_needs_allreduce_fusion", False - ): - if apply_flashinfer_allreduce_fusion(hidden_states.shape[0]) and hasattr( - norm, "forward_with_allreduce_fusion" - ): - return norm.forward_with_allreduce_fusion( - hidden_states, residual, use_attn_tp_group=False - ) - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - return norm(hidden_states, residual) - - if residual is None: - residual = hidden_states - hidden_states = norm(hidden_states) - return hidden_states, residual - return norm(hidden_states, residual) diff --git a/test/registered/unit/layers/test_linear_attention_masked_idle_rank.py b/test/registered/unit/layers/test_linear_attention_masked_idle_rank.py new file mode 100644 index 0000000000..293071afd7 --- /dev/null +++ b/test/registered/unit/layers/test_linear_attention_masked_idle_rank.py @@ -0,0 +1,54 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers import radix_linear_attention +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class _EmptyBatchRejectingBackend: + """Stand-in with the Blackwell GDN prefill contract: an empty batch raises.""" + + def __init__(self): + self.calls = 0 + + def forward(self, *, layer, forward_batch, mixed_qkv, a, b, linear_attn_output): + self.calls += 1 + if mixed_qkv.shape[0] == 0: + raise RuntimeError("max_seqlen must be positive, got 0") + linear_attn_output.fill_(1.0) + return linear_attn_output + + +class TestLinearAttentionMaskedIdleRank(CustomTestCase): + def test_fully_masked_extend_skips_the_kernel(self): + """An idle DP rank's fully masked extend must not launch linear attention + on an empty batch, and its output rows stay finite.""" + backend = _EmptyBatchRejectingBackend() + rows = 4 + output = torch.full((1, rows, 2, 3), float("nan")) + forward_batch = SimpleNamespace( + global_num_token_non_padded_cpu=0, out_cache_loc=torch.arange(rows) + ) + with patch.object( + radix_linear_attention, "get_attn_backend", return_value=backend + ): + radix_linear_attention._linear_attention_with_output_impl( + mixed_qkv=torch.randn(rows, 8), + a=torch.randn(rows, 2), + b=torch.randn(rows, 2), + output=output, + attention_layer=None, + forward_batch=forward_batch, + ) + self.assertEqual(backend.calls, 0) + torch.testing.assert_close(output, torch.zeros_like(output)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py b/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py new file mode 100644 index 0000000000..c978ed2f1a --- /dev/null +++ b/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py @@ -0,0 +1,96 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.srt.model_executor import forward_batch_info as fbi +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +NUM_TOKENS_PER_REQ = 4 + + +def _verify_spec_info(): + return SimpleNamespace( + num_tokens_per_req=NUM_TOKENS_PER_REQ, is_draft_input=lambda: False + ) + + +def _sync(forward_mode, num_reqs, global_num_tokens): + """Run prepare_mlp_sync_batch for one rank of a hybrid-SSM model in a DP + step padded to MAX_LEN with an extend-mode batch somewhere in the group.""" + num_tokens = num_reqs * NUM_TOKENS_PER_REQ + batch = ForwardBatch( + forward_mode=forward_mode, + batch_size=num_reqs, + input_ids=torch.zeros(num_tokens, dtype=torch.long), + req_pool_indices=torch.arange(num_reqs), + seq_lens=torch.full((num_reqs,), 9), + out_cache_loc=torch.arange(num_tokens), + seq_lens_sum=9 * num_reqs, + positions=torch.arange(num_tokens), + spec_info=_verify_spec_info(), + is_extend_in_batch=True, + global_num_tokens_cpu=list(global_num_tokens), + global_num_tokens_for_logprob_cpu=list(global_num_tokens), + global_num_tokens_gpu=torch.zeros(len(global_num_tokens), dtype=torch.int64), + ) + model_runner = MagicMock(is_draft_worker=False) + model_runner.attn_tp_sequence_sharded.return_value = False + no_prefill_graph = SimpleNamespace( + graph=SimpleNamespace( + cuda_graph_config=SimpleNamespace(prefill=SimpleNamespace(bs=[])) + ) + ) + with ( + patch.multiple( + fbi, + get_parallel=lambda: SimpleNamespace(attn_tp_size=1), + get_exec=lambda: no_prefill_graph, + _elastic_should_preserve_local_token_counts=lambda **kwargs: False, + dp_gather_slot=lambda: 0, + set_dp_buffer_len=lambda *args: None, + set_is_extend_in_batch=lambda *args: None, + _is_cpu=True, + ), + patch.object( + fbi, "mambaish_config", lambda model_config: object(), create=True + ), + patch.object( + fbi.DpPaddingMode, + "get_dp_padding_mode", + return_value=fbi.DpPaddingMode.MAX_LEN, + ), + patch.object(ForwardBatch, "_pad_inputs_to_size"), + patch( + "sglang.srt.batch_overlap.two_batch_overlap.TboForwardBatchPreparer.prepare" + ), + ): + batch.prepare_mlp_sync_batch(model_runner) + return batch + + +class TestMlpSyncHybridTargetVerify(CustomTestCase): + def test_verify_batch_keeps_its_verify_layout(self): + """In a DP step padded to MAX_LEN while another rank prefills, a hybrid-SSM + verify batch keeps its verify layout: one request per draft width.""" + batch = _sync(ForwardMode.TARGET_VERIFY, num_reqs=2, global_num_tokens=[8, 0]) + self.assertEqual(batch.forward_mode, ForwardMode.TARGET_VERIFY) + self.assertEqual(batch.batch_size, 2) + self.assertIsNone(batch.extend_seq_lens) + + def test_idle_rank_joins_as_verify_batch(self): + """An idle rank in the same verify step runs as a verify batch of the + padded width, not as a fabricated prefill.""" + batch = _sync(ForwardMode.IDLE, num_reqs=0, global_num_tokens=[0, 8]) + self.assertEqual(batch.forward_mode, ForwardMode.TARGET_VERIFY) + self.assertEqual(batch._original_forward_mode, ForwardMode.IDLE) + self.assertEqual(batch.batch_size, 8 // NUM_TOKENS_PER_REQ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_nemotron_h_aux_capture.py b/test/registered/unit/models/test_nemotron_h_aux_capture.py index a96faae2ef..9d0058f9ed 100644 --- a/test/registered/unit/models/test_nemotron_h_aux_capture.py +++ b/test/registered/unit/models/test_nemotron_h_aux_capture.py @@ -6,9 +6,9 @@ import torch from torch import nn from sglang.srt.layers import communicator as comm +from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h as model -from sglang.srt.models.nemotron_h_utils import make_layer_communicator from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -29,18 +29,23 @@ class _Norm(nn.Module): class _Mixer(nn.Module): - def __init__(self, scale): + """Row-parallel stand-in: a TP partial, reduced unless the layer skips it.""" + + def __init__(self, scale, tp=1): super().__init__() self.scale = scale + self.tp = tp def forward(self, hidden_states, **kwargs): - return hidden_states * self.scale + partial = hidden_states * (self.scale / self.tp) + return partial if should_skip_mlp_all_reduce() else partial * self.tp -def _build(pattern, tp, dp_enabled, capture): +def _build(pattern, tp, capture): + config = SimpleNamespace(hybrid_override_pattern=pattern) instance = model.NemotronHModel.__new__(model.NemotronHModel) nn.Module.__init__(instance) - instance.config = SimpleNamespace(hybrid_override_pattern=pattern) + instance.config = config instance.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True) instance.start_layer, instance.end_layer = 0, len(pattern) instance.norm_f = _Norm() @@ -51,13 +56,12 @@ def _build(pattern, tp, dp_enabled, capture): layer = cls.__new__(cls) nn.Module.__init__(layer) layer.norm = _Norm() - layer.prev_layer_is_attn = i > 0 and pattern[i - 1] in "M*" - layer.layer_communicator = make_layer_communicator( - layer.norm, for_attn=kind in "M*", is_last_layer=i == len(pattern) - 1 - ) - layer.mixer = _Mixer( - (0.5 / tp if dp_enabled else 0.5) if kind in "M*" else 0.25 - ) + if kind in "M*": + layer._init_layer_communicator(config, i) + layer.mixer = _Mixer(0.5, tp) + else: + layer._init_layer_communicator(config, i, is_sparse=False) + layer.mixer = _Mixer(0.25) if kind == "M": layer._forward_mamba = lambda h, batch, mixer=layer.mixer: mixer(h) layers.append(layer) @@ -69,7 +73,7 @@ class TestNemotronAuxCapture(CustomTestCase): def test_capture_reduces_only_its_snapshot(self): """Each auxiliary snapshot equals the full hidden state at its boundary, also under DP attention and after later norms update the residual in place.""" - for pattern in ("*-", "M-", "**-", "*"): + for pattern in ("*-", "M-", "**-", "*", "*--", "-*"): for dp_enabled, tp in ((True, 2), (True, 1), (False, 2)): with self.subTest(pattern=pattern, dp_enabled=dp_enabled, tp=tp): self._check(pattern, dp_enabled, tp) @@ -112,10 +116,10 @@ class TestNemotronAuxCapture(CustomTestCase): patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=False), patch.object(comm, "apply_aiter_all_reduce_fusion", return_value=False), ): - baseline = _build(pattern, tp, dp_enabled, False)( + baseline = _build(pattern, tp, False)( batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone() ) - output, snapshots = _build(pattern, tp, dp_enabled, True)( + output, snapshots = _build(pattern, tp, True)( batch.input_ids, torch.arange(2), batch, inputs_embeds=inputs.clone() ) torch.testing.assert_close(output, baseline) diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py index 3c6a7ce85e..f8cef452a6 100644 --- a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py @@ -8,7 +8,6 @@ from torch import nn from sglang.srt.layers import communicator as comm from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h_mtp -from sglang.srt.models.nemotron_h_utils import make_layer_communicator from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -69,8 +68,11 @@ class TestNemotronMTPReduction(CustomTestCase): layer.has_start_projections = False layer.has_end_norm = False layer.mixer = nn.Identity() - layer.layer_communicator = make_layer_communicator( - _Norm(), for_attn=False, is_last_layer=True + layer.norm = _Norm() + layer._init_layer_communicator( + SimpleNamespace(hybrid_override_pattern="*E"), + 1, + is_sparse=False, ) partial = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) residual = torch.tensor([[7.0, 3.0], [5.0, 9.0]])