[Perf] [AMD] dsv41: Skip unneeded layer 20 computation under decoder-bounded-replay (#45339)
This commit is contained in:
@@ -32,10 +32,6 @@ from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import (
|
||||
BuildCausalSwaPageIndices,
|
||||
late_layer_tail_layout,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
|
||||
CandidateBlocks,
|
||||
slice_candidate_blocks,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
|
||||
HEAD_DIM as SWAPAB_HEAD_DIM,
|
||||
)
|
||||
@@ -892,6 +888,7 @@ class DeepseekV4HipRadixBackend(
|
||||
# unbound; HIP differs only in the paged top-k (_low_ratio_index_topk below)
|
||||
forward_low_ratio_sources = DeepseekV4AttnBackend.forward_low_ratio_sources
|
||||
_forward_low_ratio_sources_cp = DeepseekV4AttnBackend._forward_low_ratio_sources_cp
|
||||
write_tail_source_kv = DeepseekV4AttnBackend.write_tail_source_kv
|
||||
low_ratio_prefill_graph = DeepseekV4AttnBackend.low_ratio_prefill_graph
|
||||
# This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py.
|
||||
# PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager.
|
||||
@@ -988,10 +985,10 @@ class DeepseekV4HipRadixBackend(
|
||||
raise NotImplementedError(
|
||||
"decoder SWA bounded replay is not wired for unified_kv_triton"
|
||||
)
|
||||
# the model switches onto it after the last kv_source layer (enter_late_layer_tail)
|
||||
# the model switches onto it at the last kv_source layer (enter_late_layer_tail)
|
||||
self.tail_forward_metadata: Optional[DSV4Metadata] = None
|
||||
# Read by the shared forward_low_ratio_sources; the ROCm tail starts after
|
||||
# the source layer, so nothing sets it.
|
||||
# The source layer whose KV write_tail_source_kv wrote for every row; cleared in
|
||||
# exit_late_layer_tail.
|
||||
self.tail_source_kv_layer_id: Optional[int] = None
|
||||
self.encoder_replay = False
|
||||
self.topk = get_spec().speculative_eagle_topk or 0
|
||||
@@ -2314,60 +2311,23 @@ class DeepseekV4HipRadixBackend(
|
||||
|
||||
def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple:
|
||||
"""Switch the late layers onto the tail metadata; exit_late_layer_tail takes the
|
||||
return value. Each request's candidate mask is cut to its tail rows."""
|
||||
return value. The source layer runs inside the tail and publishes its top-k and
|
||||
candidates on the tail rows, so nothing is carried over from the full rows."""
|
||||
assert self.tail_source_kv_layer_id is not None, (
|
||||
"write_tail_source_kv must run before enter_late_layer_tail"
|
||||
)
|
||||
tail_metadata = self.tail_forward_metadata
|
||||
assert tail_metadata is not None, "no tail metadata for this forward"
|
||||
saved = (self.forward_metadata, self.candidate_masks)
|
||||
tail = tail_metadata.late_layer_tail
|
||||
tail_lens_cpu = tail.extend_seq_lens_cpu
|
||||
if isinstance(self.candidate_masks, list) and self.candidate_masks:
|
||||
# the last tail_len rows of each request's candidate publication
|
||||
self.candidate_masks = [
|
||||
(
|
||||
None
|
||||
if mask is None
|
||||
else (
|
||||
slice_candidate_blocks(
|
||||
mask, slice(mask.ids.shape[0] - t, mask.ids.shape[0])
|
||||
)
|
||||
if isinstance(mask, CandidateBlocks)
|
||||
else mask[mask.shape[0] - t :]
|
||||
)
|
||||
)
|
||||
for mask, t in zip(self.candidate_masks, tail_lens_cpu)
|
||||
]
|
||||
# the consumers after the switch read the tail buffers, so carry the last source's rows over
|
||||
full_core = saved[0].core_attn_metadata
|
||||
tail_core = tail_metadata.core_attn_metadata
|
||||
for ratio in tail_core.low_ratios:
|
||||
for full_buf, tail_buf in (
|
||||
(
|
||||
full_core.sparse_page_indices(ratio),
|
||||
tail_core.sparse_page_indices(ratio),
|
||||
),
|
||||
(
|
||||
full_core.sparse_topk_lengths(ratio),
|
||||
tail_core.sparse_topk_lengths(ratio),
|
||||
),
|
||||
(
|
||||
full_core.sparse_raw_indices(ratio),
|
||||
tail_core.sparse_raw_indices(ratio),
|
||||
),
|
||||
):
|
||||
if full_buf is None or tail_buf is None:
|
||||
continue
|
||||
rows = tail.real_rows(full_buf)
|
||||
tail_buf[: rows.shape[0]].copy_(rows)
|
||||
if tail.pad_rows:
|
||||
tail_buf[rows.shape[0] :].fill_(0 if tail_buf.ndim == 1 else -1)
|
||||
self.forward_metadata = tail_metadata
|
||||
window = self.token_to_kv_pool.request_window
|
||||
if window is not None:
|
||||
window.activate(tail_core.request_window_layout)
|
||||
window.activate(tail_metadata.core_attn_metadata.request_window_layout)
|
||||
return saved
|
||||
|
||||
def exit_late_layer_tail(self, saved: tuple, forward_batch: ForwardBatch) -> None:
|
||||
self.forward_metadata, self.candidate_masks = saved
|
||||
self.tail_source_kv_layer_id = None
|
||||
window = self.token_to_kv_pool.request_window
|
||||
if window is not None:
|
||||
window.activate(
|
||||
|
||||
@@ -176,6 +176,7 @@ from sglang.srt.models.deepseek_v4_mhc import MHC_POST_MULT_VALUE
|
||||
from sglang.srt.models.deepseek_v4_platform_stage import (
|
||||
NonCudaDecoderMixin,
|
||||
forward_layer_fused_boundary,
|
||||
fused_boundary_attn_input,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v4_replay_tail import (
|
||||
DecoderReplayTail,
|
||||
@@ -3716,10 +3717,7 @@ class DeepseekV4Model(nn.Module):
|
||||
"decoder SWA bounded replay needs kv_source_layer_ids"
|
||||
)
|
||||
last_source = max(config.kv_source_layer_ids)
|
||||
# ROCm starts the tail after the source layer: the gfx950 fused boundary
|
||||
# runs a whole layer in one call, and the ROCm backend carries the source
|
||||
# layer's top-k and candidates into the tail itself.
|
||||
self.tail_writes_source_kv = not _is_hip
|
||||
self.tail_writes_source_kv = True
|
||||
self.late_layer_start = (
|
||||
last_source if self.tail_writes_source_kv else last_source + 1
|
||||
)
|
||||
@@ -3930,9 +3928,14 @@ class DeepseekV4Model(nn.Module):
|
||||
if tail is not None and i == self.late_layer_start:
|
||||
if self.tail_writes_source_kv:
|
||||
source = self.layers[i]
|
||||
if _is_hip and source.hc_boundary_fused:
|
||||
# The previous FFN hc_post has not run yet, so run it here too.
|
||||
source_x = fused_boundary_attn_input(source, state)
|
||||
else:
|
||||
source_x = mhc.combine(source.attn_hc, state)
|
||||
attn_backend.write_tail_source_kv(
|
||||
layer=source.self_attn,
|
||||
x=mhc.combine(source.attn_hc, state),
|
||||
x=source_x,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
@@ -639,6 +639,27 @@ def forward_layer_fused_boundary(
|
||||
)
|
||||
|
||||
|
||||
def fused_boundary_attn_input(layer, state: mhc.HcState) -> torch.Tensor:
|
||||
"""
|
||||
Compute the previous layer's mHC post and this layer's attention input for every
|
||||
row, with the kernel that starts forward_hc_pre_from_prev_fused_boundary.
|
||||
"""
|
||||
if isinstance(state.streams, mhc.HcPending):
|
||||
pending_post = tuple(state.streams)
|
||||
else:
|
||||
pending_post = (None, state.streams, None, None)
|
||||
_, x, coefficients = make_amd_hc_boundary(
|
||||
layer,
|
||||
*pending_post,
|
||||
state.pre,
|
||||
layer.hc_attn_fn,
|
||||
layer.hc_attn_scale,
|
||||
layer.hc_attn_base,
|
||||
)
|
||||
x, _ = layer._input_norm(x, allow_aiter_quant=False, coefficients=coefficients)
|
||||
return x
|
||||
|
||||
|
||||
def forward_hc_pre_from_prev_fused_boundary(
|
||||
layer,
|
||||
positions: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user