[AMD] Use AITER ASM prefill for MiniMax-M3 HD128 attention (#41707)
Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com> Co-authored-by: Thomas Wang <thomawan@amd.com>
This commit is contained in:
co-authored by
Zijie Chen
Thomas Wang
parent
4ab720e655
commit
71dc292b97
@@ -579,6 +579,8 @@ class Envs:
|
||||
# of the occupancy-starved mha_batch_prefill FMHA. Independent kill-switch
|
||||
# for the new path; pairs with SGLANG_AITER_UNIFIED_VERIFY. Default on.
|
||||
SGLANG_AITER_UNIFIED_DRAFT_EXTEND = EnvBool(True)
|
||||
# Use ASM prefill for gfx950 HD128 FP8 KV, including cached prefixes.
|
||||
SGLANG_AITER_ASM_PREFILL_HD128 = EnvBool(True)
|
||||
# Attention (aiter, ROCm): hand chunked prefill the page-level KV view so
|
||||
# gfx950 fp8 hd256 takes aiter's paged-varlen asm kernel. That kernel is
|
||||
# compiled for 4D LINEAR [N, 64, H, D], so it serves --page-size 64 and
|
||||
|
||||
@@ -3735,6 +3735,58 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
envs.SGLANG_AITER_ASM_PREFILL_HD128.get()
|
||||
and is_gfx95_supported()
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
and not layer.is_cross_attention
|
||||
and window_size == (-1, -1)
|
||||
and sinks is None
|
||||
and self.logits_soft_cap == 0.0
|
||||
and layer.qk_head_dim == layer.v_head_dim == 128
|
||||
and layer.tp_k_head_num == layer.tp_v_head_num
|
||||
and self.kv_cache_dtype == fp8_dtype
|
||||
and q.dtype == torch.bfloat16
|
||||
and _aiter_fp8_asm_supports_gqa(
|
||||
layer.tp_q_head_num, layer.tp_k_head_num
|
||||
)
|
||||
and not self.kv_cache_is_vectorized_5d
|
||||
and self.forward_metadata.max_kv_len is not None
|
||||
):
|
||||
k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
|
||||
tok_idx, cu_k = self._asm_context_prefill_indices(
|
||||
forward_batch, forward_batch.batch_size, k_cache.shape[0]
|
||||
)
|
||||
if tok_idx is not None:
|
||||
# Read the already quantized cache for first and later
|
||||
# chunks alike. Raw K/V may come from different producers;
|
||||
# casting them here would skip or repeat their KV scaling.
|
||||
k_gather = (
|
||||
k_cache.view(torch.uint8)
|
||||
.index_select(0, tok_idx)
|
||||
.view(fp8_dtype)
|
||||
)
|
||||
v_gather = (
|
||||
v_cache.view(torch.uint8)
|
||||
.index_select(0, tok_idx)
|
||||
.view(fp8_dtype)
|
||||
)
|
||||
o = flash_attn_varlen_fp8_pertensor_func(
|
||||
q.contiguous().view(-1, layer.tp_q_head_num, 128).to(fp8_dtype),
|
||||
k_gather,
|
||||
v_gather,
|
||||
self.k_scale.reshape(1), # Q is cast at unit scale.
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
cu_k,
|
||||
self.forward_metadata.max_q_len,
|
||||
int(self.forward_metadata.max_kv_len),
|
||||
softmax_scale=layer.scaling,
|
||||
causal=True,
|
||||
)
|
||||
return o.to(self.input_dtype).view(-1, layer.tp_q_head_num * 128)
|
||||
|
||||
# Context-chunk prefill (extend batches WITH a prefix) via the
|
||||
# gfx950 ASM fp8 varlen fmha. The ck_tile paged batch_prefill runs
|
||||
# at ~15% FP8 MFU at these shapes while the ASM kernel is ~3.5x
|
||||
|
||||
Reference in New Issue
Block a user