[ROCm][Perf] aiter: page-level KV view for gfx950 fp8 page-64 asm prefill (#36505)
This commit is contained in:
@@ -576,6 +576,16 @@ class Envs:
|
||||
# of the occupancy-starved mha_batch_prefill FMHA. Independent kill-switch
|
||||
# for the new path; pairs with SGLANG_AITER_UNIFIED_VERIFY. Default on.
|
||||
SGLANG_AITER_UNIFIED_DRAFT_EXTEND = EnvBool(True)
|
||||
# Attention (aiter, ROCm): hand chunked prefill the page-level KV view so
|
||||
# gfx950 fp8 hd256 takes aiter's paged-varlen asm kernel. That kernel is
|
||||
# compiled for 4D LINEAR [N, 64, H, D], so it serves --page-size 64 and
|
||||
# nothing else; at any other page size the view is never built and this
|
||||
# flag does nothing whichever way it is set. Where it does apply the win
|
||||
# grows with context (~1-2% throughput and 3-7% TTFT at 60k and above) and
|
||||
# is flat to marginally negative at moderate ISL, so this is a per-workload
|
||||
# switch rather than a pure kill-switch. Off falls back to the contiguous
|
||||
# gather path. Default on.
|
||||
SGLANG_AITER_PAGED_PREFILL_ASM = EnvBool(True)
|
||||
# size the KV pool after CUDA-graph capture
|
||||
SGLANG_ENABLE_POST_CAPTURE_KV_SIZING = EnvBool(False)
|
||||
|
||||
|
||||
@@ -164,6 +164,44 @@ class ForwardMetadata:
|
||||
asm_ctx_ready: bool = False
|
||||
asm_ctx_tok_idx: Optional[torch.Tensor] = None
|
||||
asm_ctx_cu_k: Optional[torch.Tensor] = None
|
||||
# (page_indptr, page_ids, last_page_len); None means use the token-level table
|
||||
paged_kv_view: Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None
|
||||
|
||||
|
||||
def _build_paged_kv_view(
|
||||
kv_indices: torch.Tensor, seq_lens_cpu: torch.Tensor, page_size: int
|
||||
):
|
||||
"""Token-level flashinfer KV table -> the page-level view aiter's asm
|
||||
paged-varlen prefill expects.
|
||||
|
||||
The paged allocator packs each page contiguously, so a token slot's page id
|
||||
is slot // page_size. The arithmetic runs on seq_lens_cpu so it costs no
|
||||
device sync; only the gather touches the GPU.
|
||||
"""
|
||||
device = kv_indices.device
|
||||
seq_lens = seq_lens_cpu.to(torch.int64)
|
||||
pages = (seq_lens + page_size - 1) // page_size
|
||||
|
||||
page_indptr = torch.zeros(pages.numel() + 1, dtype=torch.int32)
|
||||
page_indptr[1:] = torch.cumsum(pages, dim=0)
|
||||
tok_base = torch.zeros(pages.numel() + 1, dtype=torch.int64)
|
||||
tok_base[1:] = torch.cumsum(seq_lens, dim=0)
|
||||
|
||||
within = torch.arange(
|
||||
int(page_indptr[-1]), dtype=torch.int64
|
||||
) - torch.repeat_interleave(page_indptr[:-1].to(torch.int64), pages)
|
||||
gather = torch.repeat_interleave(tok_base[:-1], pages) + within * page_size
|
||||
page_ids = (kv_indices[gather.to(device)] // page_size).to(torch.int32)
|
||||
last_page_len = ((seq_lens - 1) % page_size + 1).to(torch.int32)
|
||||
return page_indptr.to(device), page_ids, last_page_len.to(device)
|
||||
|
||||
|
||||
def _paged_prefill_asm_supports_gqa(num_q_heads: int, num_kv_heads: int) -> bool:
|
||||
"""aiter's asm paged-varlen guard takes only a power-of-two GQA ratio."""
|
||||
if num_kv_heads <= 0 or num_q_heads % num_kv_heads != 0:
|
||||
return False
|
||||
gqa = num_q_heads // num_kv_heads
|
||||
return gqa & (gqa - 1) == 0
|
||||
|
||||
|
||||
_AITER_PARTITION_SIZE_ROCM = 256
|
||||
@@ -2030,6 +2068,25 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.indices_updater_prefill.kv_indices
|
||||
).to(torch.int32)
|
||||
|
||||
# Once per batch, not per layer: forward_extend only consumes it.
|
||||
# The arch test is not redundant with the flag: the asm guard is
|
||||
# gfx95-only, and elsewhere there is no kernel for this shape at
|
||||
# all, so building a view we must not pass is wasted work.
|
||||
paged_kv_view = None
|
||||
if (
|
||||
envs.SGLANG_AITER_PAGED_PREFILL_ASM.get()
|
||||
and is_gfx95_supported()
|
||||
and self.page_size == 64
|
||||
and not self.kv_cache_is_vectorized_5d
|
||||
and not self.use_sliding_window_kv_pool
|
||||
and not self.use_triton_unified_attention
|
||||
):
|
||||
paged_kv_view = _build_paged_kv_view(
|
||||
self.indices_updater_prefill.kv_indices,
|
||||
forward_batch.seq_lens_cpu,
|
||||
self.page_size,
|
||||
)
|
||||
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
self.indices_updater_prefill.kv_indptr,
|
||||
self.indices_updater_prefill.kv_indices,
|
||||
@@ -2039,6 +2096,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
forward_batch.seq_lens_cpu.max().item(),
|
||||
swa_page_table=swa_page_table,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
paged_kv_view=paged_kv_view,
|
||||
)
|
||||
|
||||
def _plan_dcp_decode_metadata(
|
||||
@@ -3543,6 +3601,18 @@ class AiterAttnBackend(AttentionBackend):
|
||||
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
|
||||
window_size = (layer.sliding_window_size, -1)
|
||||
|
||||
# Whether the paged asm prefill kernel can serve this layer on this
|
||||
# batch. Both the gather branch below and the paged branch at the
|
||||
# end of this method key off it, so the two stay mutually exclusive
|
||||
# by construction: one flag, read twice, cannot drift out of sync
|
||||
# the way two copies of the same condition would.
|
||||
paged_asm_available = (
|
||||
self.forward_metadata.paged_kv_view is not None
|
||||
and _paged_prefill_asm_supports_gqa(
|
||||
layer.tp_q_head_num, layer.tp_k_head_num
|
||||
)
|
||||
)
|
||||
|
||||
# Context-chunk prefill (extend batches WITH a prefix) via the
|
||||
# gfx950 ASM fp8 varlen fmha. The ck_tile paged batch_prefill runs
|
||||
# at ~15% FP8 MFU at these shapes while the ASM kernel is ~3.5x
|
||||
@@ -3551,6 +3621,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# first chunk already takes the ASM branch below.
|
||||
# This applies to Qwen3.5 full-attention layers only currently,
|
||||
# Other configurations fall through to the attention paths below.
|
||||
#
|
||||
# Yields to the paged asm kernel when there is one, since that reads
|
||||
# the cache in place and does the same attention with no gather at
|
||||
# all. This is not a blanket disable: on any batch or layer that
|
||||
# kernel cannot serve, the flag is False and this branch runs
|
||||
# exactly as it does today.
|
||||
if (
|
||||
is_gfx95_supported()
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
@@ -3567,6 +3643,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
and not self.kv_cache_is_vectorized_5d
|
||||
and self.forward_metadata.max_kv_len is not None
|
||||
and not paged_asm_available
|
||||
):
|
||||
bs = forward_batch.batch_size
|
||||
k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
|
||||
@@ -3704,12 +3781,36 @@ class AiterAttnBackend(AttentionBackend):
|
||||
-1, layer.tp_q_head_num, layer.head_dim
|
||||
)
|
||||
|
||||
kv_indptr_arg = self.forward_metadata.kv_indptr[:bs0]
|
||||
if (
|
||||
paged_asm_available
|
||||
and page_table is self.forward_metadata.kv_indices
|
||||
and window_size == (-1, -1)
|
||||
and sinks is None
|
||||
and self.logits_soft_cap == 0.0
|
||||
and layer.qk_head_dim == 256
|
||||
and layer.v_head_dim == 256
|
||||
and self.kv_cache_dtype == fp8_dtype
|
||||
):
|
||||
# These must match aiter's asm guard: there is no CK arm for 4D
|
||||
# LINEAR page-64 fp8 hd256, so a shape the guard rejects raises
|
||||
# rather than falling back. The arch half of the guard is
|
||||
# checked where paged_kv_view is built.
|
||||
page_indptr, page_ids, last_page_len = (
|
||||
self.forward_metadata.paged_kv_view
|
||||
)
|
||||
kv_indptr_arg = page_indptr[:bs0]
|
||||
page_table = page_ids
|
||||
k_cache = k_cache.view(-1, self.page_size, *k_cache.shape[-2:])
|
||||
v_cache = v_cache.view(-1, self.page_size, *v_cache.shape[-2:])
|
||||
extra_kwargs["kv_last_page_lens"] = last_page_len[: bs0 - 1]
|
||||
|
||||
o = mha_batch_prefill_func(
|
||||
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_cache,
|
||||
v_cache,
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.kv_indptr[:bs0],
|
||||
kv_indptr_arg,
|
||||
page_table,
|
||||
self.forward_metadata.max_q_len,
|
||||
self.forward_metadata.max_kv_len,
|
||||
|
||||
Reference in New Issue
Block a user