[AMD] Use AITER ASM prefill for MiniMax-M3 HD128 attention (#41707)

Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com>
Co-authored-by: Thomas Wang <thomawan@amd.com>
This commit is contained in:
zijie
2026-10-03 17:54:49 -07:00
committed by GitHub
co-authored by Zijie Chen Thomas Wang
parent 4ab720e655
commit 71dc292b97
3 changed files with 211 additions and 0 deletions
+2
View File
@@ -579,6 +579,8 @@ class Envs:
# of the occupancy-starved mha_batch_prefill FMHA. Independent kill-switch
# for the new path; pairs with SGLANG_AITER_UNIFIED_VERIFY. Default on.
SGLANG_AITER_UNIFIED_DRAFT_EXTEND = EnvBool(True)
# Use ASM prefill for gfx950 HD128 FP8 KV, including cached prefixes.
SGLANG_AITER_ASM_PREFILL_HD128 = EnvBool(True)
# Attention (aiter, ROCm): hand chunked prefill the page-level KV view so
# gfx950 fp8 hd256 takes aiter's paged-varlen asm kernel. That kernel is
# compiled for 4D LINEAR [N, 64, H, D], so it serves --page-size 64 and
@@ -3735,6 +3735,58 @@ class AiterAttnBackend(AttentionBackend):
)
)
if (
envs.SGLANG_AITER_ASM_PREFILL_HD128.get()
and is_gfx95_supported()
and forward_batch.forward_mode.is_extend()
and not layer.is_cross_attention
and window_size == (-1, -1)
and sinks is None
and self.logits_soft_cap == 0.0
and layer.qk_head_dim == layer.v_head_dim == 128
and layer.tp_k_head_num == layer.tp_v_head_num
and self.kv_cache_dtype == fp8_dtype
and q.dtype == torch.bfloat16
and _aiter_fp8_asm_supports_gqa(
layer.tp_q_head_num, layer.tp_k_head_num
)
and not self.kv_cache_is_vectorized_5d
and self.forward_metadata.max_kv_len is not None
):
k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
tok_idx, cu_k = self._asm_context_prefill_indices(
forward_batch, forward_batch.batch_size, k_cache.shape[0]
)
if tok_idx is not None:
# Read the already quantized cache for first and later
# chunks alike. Raw K/V may come from different producers;
# casting them here would skip or repeat their KV scaling.
k_gather = (
k_cache.view(torch.uint8)
.index_select(0, tok_idx)
.view(fp8_dtype)
)
v_gather = (
v_cache.view(torch.uint8)
.index_select(0, tok_idx)
.view(fp8_dtype)
)
o = flash_attn_varlen_fp8_pertensor_func(
q.contiguous().view(-1, layer.tp_q_head_num, 128).to(fp8_dtype),
k_gather,
v_gather,
self.k_scale.reshape(1), # Q is cast at unit scale.
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
cu_k,
self.forward_metadata.max_q_len,
int(self.forward_metadata.max_kv_len),
softmax_scale=layer.scaling,
causal=True,
)
return o.to(self.input_dtype).view(-1, layer.tp_q_head_num * 128)
# Context-chunk prefill (extend batches WITH a prefix) via the
# gfx950 ASM fp8 varlen fmha. The ck_tile paged batch_prefill runs
# at ~15% FP8 MFU at these shapes while the ASM kernel is ~3.5x