[Perf] [AMD] dsv41: Skip unneeded layer 20 computation under decoder-bounded-replay (#45339)

This commit is contained in:
Thomas Wang
2026-10-20 11:49:05 -07:00
committed by GitHub
parent f82465718b
commit 82ac83a8d0
3 changed files with 40 additions and 56 deletions
@@ -32,10 +32,6 @@ from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import (
BuildCausalSwaPageIndices,
late_layer_tail_layout,
)
from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
CandidateBlocks,
slice_candidate_blocks,
)
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
HEAD_DIM as SWAPAB_HEAD_DIM,
)
@@ -892,6 +888,7 @@ class DeepseekV4HipRadixBackend(
# unbound; HIP differs only in the paged top-k (_low_ratio_index_topk below)
forward_low_ratio_sources = DeepseekV4AttnBackend.forward_low_ratio_sources
_forward_low_ratio_sources_cp = DeepseekV4AttnBackend._forward_low_ratio_sources_cp
write_tail_source_kv = DeepseekV4AttnBackend.write_tail_source_kv
low_ratio_prefill_graph = DeepseekV4AttnBackend.low_ratio_prefill_graph
# This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py.
# PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager.
@@ -988,10 +985,10 @@ class DeepseekV4HipRadixBackend(
raise NotImplementedError(
"decoder SWA bounded replay is not wired for unified_kv_triton"
)
# the model switches onto it after the last kv_source layer (enter_late_layer_tail)
# the model switches onto it at the last kv_source layer (enter_late_layer_tail)
self.tail_forward_metadata: Optional[DSV4Metadata] = None
# Read by the shared forward_low_ratio_sources; the ROCm tail starts after
# the source layer, so nothing sets it.
# The source layer whose KV write_tail_source_kv wrote for every row; cleared in
# exit_late_layer_tail.
self.tail_source_kv_layer_id: Optional[int] = None
self.encoder_replay = False
self.topk = get_spec().speculative_eagle_topk or 0
@@ -2314,60 +2311,23 @@ class DeepseekV4HipRadixBackend(
def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple:
"""Switch the late layers onto the tail metadata; exit_late_layer_tail takes the
return value. Each request's candidate mask is cut to its tail rows."""
return value. The source layer runs inside the tail and publishes its top-k and
candidates on the tail rows, so nothing is carried over from the full rows."""
assert self.tail_source_kv_layer_id is not None, (
"write_tail_source_kv must run before enter_late_layer_tail"
)
tail_metadata = self.tail_forward_metadata
assert tail_metadata is not None, "no tail metadata for this forward"
saved = (self.forward_metadata, self.candidate_masks)
tail = tail_metadata.late_layer_tail
tail_lens_cpu = tail.extend_seq_lens_cpu
if isinstance(self.candidate_masks, list) and self.candidate_masks:
# the last tail_len rows of each request's candidate publication
self.candidate_masks = [
(
None
if mask is None
else (
slice_candidate_blocks(
mask, slice(mask.ids.shape[0] - t, mask.ids.shape[0])
)
if isinstance(mask, CandidateBlocks)
else mask[mask.shape[0] - t :]
)
)
for mask, t in zip(self.candidate_masks, tail_lens_cpu)
]
# the consumers after the switch read the tail buffers, so carry the last source's rows over
full_core = saved[0].core_attn_metadata
tail_core = tail_metadata.core_attn_metadata
for ratio in tail_core.low_ratios:
for full_buf, tail_buf in (
(
full_core.sparse_page_indices(ratio),
tail_core.sparse_page_indices(ratio),
),
(
full_core.sparse_topk_lengths(ratio),
tail_core.sparse_topk_lengths(ratio),
),
(
full_core.sparse_raw_indices(ratio),
tail_core.sparse_raw_indices(ratio),
),
):
if full_buf is None or tail_buf is None:
continue
rows = tail.real_rows(full_buf)
tail_buf[: rows.shape[0]].copy_(rows)
if tail.pad_rows:
tail_buf[rows.shape[0] :].fill_(0 if tail_buf.ndim == 1 else -1)
self.forward_metadata = tail_metadata
window = self.token_to_kv_pool.request_window
if window is not None:
window.activate(tail_core.request_window_layout)
window.activate(tail_metadata.core_attn_metadata.request_window_layout)
return saved
def exit_late_layer_tail(self, saved: tuple, forward_batch: ForwardBatch) -> None:
self.forward_metadata, self.candidate_masks = saved
self.tail_source_kv_layer_id = None
window = self.token_to_kv_pool.request_window
if window is not None:
window.activate(
+8 -5
View File
@@ -176,6 +176,7 @@ from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE
from sglang.srt.models.deepseek_v4_platform_stage import (
NonCudaDecoderMixin,
forward_layer_fused_boundary,
fused_boundary_attn_input,
)
from sglang.srt.models.deepseek_v4_replay_tail import (
DecoderReplayTail,
@@ -3716,10 +3717,7 @@ class DeepseekV4Model(nn.Module):
"decoder SWA bounded replay needs kv_source_layer_ids"
)
last_source = max(config.kv_source_layer_ids)
# ROCm starts the tail after the source layer: the gfx950 fused boundary
# runs a whole layer in one call, and the ROCm backend carries the source
# layer's top-k and candidates into the tail itself.
self.tail_writes_source_kv = not _is_hip
self.tail_writes_source_kv = True
self.late_layer_start = (
last_source if self.tail_writes_source_kv else last_source + 1
)
@@ -3930,9 +3928,14 @@ class DeepseekV4Model(nn.Module):
if tail is not None and i == self.late_layer_start:
if self.tail_writes_source_kv:
source = self.layers[i]
if _is_hip and source.hc_boundary_fused:
# The previous FFN hc_post has not run yet, so run it here too.
source_x = fused_boundary_attn_input(source, state)
else:
source_x = mhc.combine(source.attn_hc, state)
attn_backend.write_tail_source_kv(
layer=source.self_attn,
x=mhc.combine(source.attn_hc, state),
x=source_x,
positions=positions,
forward_batch=forward_batch,
)
@@ -639,6 +639,27 @@ def forward_layer_fused_boundary(
)
def fused_boundary_attn_input(layer, state: mhc.HcState) -> torch.Tensor:
"""
Compute the previous layer's mHC post and this layer's attention input for every
row, with the kernel that starts forward_hc_pre_from_prev_fused_boundary.
"""
if isinstance(state.streams, mhc.HcPending):
pending_post = tuple(state.streams)
else:
pending_post = (None, state.streams, None, None)
_, x, coefficients = make_amd_hc_boundary(
layer,
*pending_post,
state.pre,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
)
x, _ = layer._input_norm(x, allow_aiter_quant=False, coefficients=coefficients)
return x
def forward_hc_pre_from_prev_fused_boundary(
layer,
positions: torch.Tensor,