From 82ac83a8d0ab50ea7c9fe8e5b5934c2fa5a224bf Mon Sep 17 00:00:00 2001 From: Thomas Wang Date: Wed, 21 Oct 2026 02:49:05 +0800 Subject: [PATCH] [Perf] [AMD] dsv41: Skip unneeded layer 20 computation under decoder-bounded-replay (#45339) --- .../deepseek_v4_backend_hip_radix.py | 62 ++++--------------- python/sglang/srt/models/deepseek_v4.py | 13 ++-- .../srt/models/deepseek_v4_platform_stage.py | 21 +++++++ 3 files changed, 40 insertions(+), 56 deletions(-) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index a71891e512..2d05f6b711 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -32,10 +32,6 @@ from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( BuildCausalSwaPageIndices, late_layer_tail_layout, ) -from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( - CandidateBlocks, - slice_candidate_blocks, -) from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( HEAD_DIM as SWAPAB_HEAD_DIM, ) @@ -892,6 +888,7 @@ class DeepseekV4HipRadixBackend( # unbound; HIP differs only in the paged top-k (_low_ratio_index_topk below) forward_low_ratio_sources = DeepseekV4AttnBackend.forward_low_ratio_sources _forward_low_ratio_sources_cp = DeepseekV4AttnBackend._forward_low_ratio_sources_cp + write_tail_source_kv = DeepseekV4AttnBackend.write_tail_source_kv low_ratio_prefill_graph = DeepseekV4AttnBackend.low_ratio_prefill_graph # This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py. # PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager. @@ -988,10 +985,10 @@ class DeepseekV4HipRadixBackend( raise NotImplementedError( "decoder SWA bounded replay is not wired for unified_kv_triton" ) - # the model switches onto it after the last kv_source layer (enter_late_layer_tail) + # the model switches onto it at the last kv_source layer (enter_late_layer_tail) self.tail_forward_metadata: Optional[DSV4Metadata] = None - # Read by the shared forward_low_ratio_sources; the ROCm tail starts after - # the source layer, so nothing sets it. + # The source layer whose KV write_tail_source_kv wrote for every row; cleared in + # exit_late_layer_tail. self.tail_source_kv_layer_id: Optional[int] = None self.encoder_replay = False self.topk = get_spec().speculative_eagle_topk or 0 @@ -2314,60 +2311,23 @@ class DeepseekV4HipRadixBackend( def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple: """Switch the late layers onto the tail metadata; exit_late_layer_tail takes the - return value. Each request's candidate mask is cut to its tail rows.""" + return value. The source layer runs inside the tail and publishes its top-k and + candidates on the tail rows, so nothing is carried over from the full rows.""" + assert self.tail_source_kv_layer_id is not None, ( + "write_tail_source_kv must run before enter_late_layer_tail" + ) tail_metadata = self.tail_forward_metadata assert tail_metadata is not None, "no tail metadata for this forward" saved = (self.forward_metadata, self.candidate_masks) - tail = tail_metadata.late_layer_tail - tail_lens_cpu = tail.extend_seq_lens_cpu - if isinstance(self.candidate_masks, list) and self.candidate_masks: - # the last tail_len rows of each request's candidate publication - self.candidate_masks = [ - ( - None - if mask is None - else ( - slice_candidate_blocks( - mask, slice(mask.ids.shape[0] - t, mask.ids.shape[0]) - ) - if isinstance(mask, CandidateBlocks) - else mask[mask.shape[0] - t :] - ) - ) - for mask, t in zip(self.candidate_masks, tail_lens_cpu) - ] - # the consumers after the switch read the tail buffers, so carry the last source's rows over - full_core = saved[0].core_attn_metadata - tail_core = tail_metadata.core_attn_metadata - for ratio in tail_core.low_ratios: - for full_buf, tail_buf in ( - ( - full_core.sparse_page_indices(ratio), - tail_core.sparse_page_indices(ratio), - ), - ( - full_core.sparse_topk_lengths(ratio), - tail_core.sparse_topk_lengths(ratio), - ), - ( - full_core.sparse_raw_indices(ratio), - tail_core.sparse_raw_indices(ratio), - ), - ): - if full_buf is None or tail_buf is None: - continue - rows = tail.real_rows(full_buf) - tail_buf[: rows.shape[0]].copy_(rows) - if tail.pad_rows: - tail_buf[rows.shape[0] :].fill_(0 if tail_buf.ndim == 1 else -1) self.forward_metadata = tail_metadata window = self.token_to_kv_pool.request_window if window is not None: - window.activate(tail_core.request_window_layout) + window.activate(tail_metadata.core_attn_metadata.request_window_layout) return saved def exit_late_layer_tail(self, saved: tuple, forward_batch: ForwardBatch) -> None: self.forward_metadata, self.candidate_masks = saved + self.tail_source_kv_layer_id = None window = self.token_to_kv_pool.request_window if window is not None: window.activate( diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 00579d167e..507a1b207c 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -176,6 +176,7 @@ from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE from sglang.srt.models.deepseek_v4_platform_stage import ( NonCudaDecoderMixin, forward_layer_fused_boundary, + fused_boundary_attn_input, ) from sglang.srt.models.deepseek_v4_replay_tail import ( DecoderReplayTail, @@ -3716,10 +3717,7 @@ class DeepseekV4Model(nn.Module): "decoder SWA bounded replay needs kv_source_layer_ids" ) last_source = max(config.kv_source_layer_ids) - # ROCm starts the tail after the source layer: the gfx950 fused boundary - # runs a whole layer in one call, and the ROCm backend carries the source - # layer's top-k and candidates into the tail itself. - self.tail_writes_source_kv = not _is_hip + self.tail_writes_source_kv = True self.late_layer_start = ( last_source if self.tail_writes_source_kv else last_source + 1 ) @@ -3930,9 +3928,14 @@ class DeepseekV4Model(nn.Module): if tail is not None and i == self.late_layer_start: if self.tail_writes_source_kv: source = self.layers[i] + if _is_hip and source.hc_boundary_fused: + # The previous FFN hc_post has not run yet, so run it here too. + source_x = fused_boundary_attn_input(source, state) + else: + source_x = mhc.combine(source.attn_hc, state) attn_backend.write_tail_source_kv( layer=source.self_attn, - x=mhc.combine(source.attn_hc, state), + x=source_x, positions=positions, forward_batch=forward_batch, ) diff --git a/python/sglang/srt/models/deepseek_v4_platform_stage.py b/python/sglang/srt/models/deepseek_v4_platform_stage.py index 2734acf78f..b9813675e0 100644 --- a/python/sglang/srt/models/deepseek_v4_platform_stage.py +++ b/python/sglang/srt/models/deepseek_v4_platform_stage.py @@ -639,6 +639,27 @@ def forward_layer_fused_boundary( ) +def fused_boundary_attn_input(layer, state: mhc.HcState) -> torch.Tensor: + """ + Compute the previous layer's mHC post and this layer's attention input for every + row, with the kernel that starts forward_hc_pre_from_prev_fused_boundary. + """ + if isinstance(state.streams, mhc.HcPending): + pending_post = tuple(state.streams) + else: + pending_post = (None, state.streams, None, None) + _, x, coefficients = make_amd_hc_boundary( + layer, + *pending_post, + state.pre, + layer.hc_attn_fn, + layer.hc_attn_scale, + layer.hc_attn_base, + ) + x, _ = layer._input_norm(x, allow_aiter_quant=False, coefficients=coefficients) + return x + + def forward_hc_pre_from_prev_fused_boundary( layer, positions: torch.Tensor,