[Fix] [AMD] dsa: Fix AITER prefill with an FP8 KV cache (#39083)

Co-authored-by: Pawel Kowalski <pawel.kowalski@silo.ai>
Co-authored-by: Thomas Wang <thomawan@amd.com>
This commit is contained in:
Pawel Kowalski
2026-10-23 12:30:42 -07:00
committed by GitHub
co-authored by Pawel Kowalski Thomas Wang
parent e6451c1a95
commit 0931dfedc9
2 changed files with 37 additions and 19 deletions
@@ -644,6 +644,9 @@ class DeepseekSparseAttnBackend(
self.aiter_dsa_metadata_kv_dtype = None
self.aiter_dsa_kv_last_page_lens = None
self.aiter_dsa_work_metadata = None
self.aiter_dsa_identity_scale = torch.ones(
(), dtype=torch.float32, device=self.device
)
if (
self.dsa_prefill_impl == "aiter" or self.dsa_decode_impl == "aiter"
@@ -802,9 +805,8 @@ class DeepseekSparseAttnBackend(
q_dtype,
kv_dtype,
is_sparse=True,
fast_mode=False,
num_kv_splits=self.aiter_dsa_max_split_per_batch,
intra_batch_mode=True,
fast_mode=True,
intra_batch_mode=False,
)
return (
@@ -900,10 +902,10 @@ class DeepseekSparseAttnBackend(
kv_granularity=16,
max_seqlen_qo=max_seqlen_q,
uni_seqlen_qo=max_seqlen_q,
fast_mode=False,
topk=self.dsa_index_topk,
fast_mode=True,
topk=-1,
max_split_per_batch=self.aiter_dsa_max_split_per_batch,
intra_batch_mode=True,
intra_batch_mode=False,
dtype_q=q_dtype,
dtype_kv=kv_dtype,
)
@@ -916,8 +918,6 @@ class DeepseekSparseAttnBackend(
"reduce_indptr": self.aiter_dsa_reduce_indptr,
"reduce_final_map": self.aiter_dsa_reduce_final_map,
"reduce_partial_map": self.aiter_dsa_reduce_partial_map,
"intra_batch_mode": True,
"num_kv_splits": self.aiter_dsa_max_split_per_batch,
}
def _pad_trtllm_sparse_page_table(
@@ -3620,11 +3620,14 @@ class DeepseekSparseAttnBackend(
bs: int,
) -> torch.Tensor:
q = q_all.reshape(-1, layer.tp_q_head_num * layer.head_dim)
o_dtype = torch.bfloat16 if q.dtype == fp8_dtype else q.dtype
if layer.head_dim != layer.v_head_dim:
o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim))
o = q.new_empty(
(q.shape[0], layer.tp_q_head_num * layer.v_head_dim), dtype=o_dtype
)
else:
o = torch.empty_like(q)
o = torch.empty_like(q, dtype=o_dtype)
if self.need_pad_heads:
q_kernel = q.view(
@@ -3635,7 +3638,8 @@ class DeepseekSparseAttnBackend(
q.shape[0],
layer.tp_q_head_num * self.head_repeat_factor,
layer.v_head_dim,
)
),
dtype=o_dtype,
)
else:
q_kernel = q.view(-1, layer.tp_q_head_num, layer.head_dim)
@@ -3645,7 +3649,12 @@ class DeepseekSparseAttnBackend(
kv_scale = None
aiter_persistent_kwargs = {}
if kv_cache.dtype == fp8_dtype:
kv_scale = torch.ones((), dtype=torch.float32, device=q_kernel.device)
q_scale = self.aiter_dsa_identity_scale
kv_scale = (
layer.k_scale.reshape(())
if isinstance(layer.k_scale, torch.Tensor)
else self.aiter_dsa_identity_scale
)
kv_indptr = self.kv_indptr
@@ -3698,11 +3707,14 @@ class DeepseekSparseAttnBackend(
) -> torch.Tensor:
num_tokens = q_all.shape[0]
q = q_all.reshape(-1, layer.tp_q_head_num * layer.head_dim)
o_dtype = torch.bfloat16 if q.dtype == fp8_dtype else q.dtype
if layer.head_dim != layer.v_head_dim:
o = q.new_empty((num_tokens, layer.tp_q_head_num * layer.v_head_dim))
o = q.new_empty(
(num_tokens, layer.tp_q_head_num * layer.v_head_dim), dtype=o_dtype
)
else:
o = torch.empty_like(q)
o = torch.empty_like(q, dtype=o_dtype)
if self.need_pad_heads:
q_kernel = q.view(
@@ -3713,7 +3725,8 @@ class DeepseekSparseAttnBackend(
num_tokens,
layer.tp_q_head_num * self.head_repeat_factor,
layer.v_head_dim,
)
),
dtype=o_dtype,
)
else:
q_kernel = q.view(-1, layer.tp_q_head_num, layer.head_dim)
@@ -3723,7 +3736,12 @@ class DeepseekSparseAttnBackend(
kv_scale = None
aiter_persistent_kwargs = {}
if kv_cache.dtype == fp8_dtype:
kv_scale = torch.ones((), dtype=torch.float32, device=q_kernel.device)
q_scale = self.aiter_dsa_identity_scale
kv_scale = (
layer.k_scale.reshape(())
if isinstance(layer.k_scale, torch.Tensor)
else self.aiter_dsa_identity_scale
)
non_minus1_mask = page_table_1 != -1
non_minus1_counts = non_minus1_mask.sum(dim=1)
@@ -1248,7 +1248,7 @@ DSA_DECODE_IMPL_VARIANTS: tuple[str, ...] = (
# take in FP8 deployments. The `flashmla_kv` decode kernel and *both*
# flashmla prefill kernels are the production-relevant FP8 paths.
DSA_FP8_COMPATIBLE_PREFILL_IMPLS: frozenset[str] = frozenset(
{"flashmla_sparse", "flashmla_kv", "flashmla_auto", "triton_sparse_mla"}
{"aiter", "flashmla_sparse", "flashmla_kv", "flashmla_auto", "triton_sparse_mla"}
)
DSA_FP8_COMPATIBLE_DECODE_IMPLS: frozenset[str] = frozenset(
{"flashmla_kv", "flashmla_auto"}
@@ -1395,8 +1395,8 @@ def run_dsa_sparse_fp8_prefill_case(
if dsa_prefill_backend not in DSA_FP8_COMPATIBLE_PREFILL_IMPLS:
testcase.skipTest(
f"DSA prefill impl `{dsa_prefill_backend}` does not support FP8 KV "
f"cache (only `flashmla_sparse`, `flashmla_kv`, and `flashmla_auto` "
f"read FP8 K directly; others require BF16 K)."
f"cache (only `aiter`, `flashmla_sparse`, `flashmla_kv`, and "
f"`flashmla_auto` read FP8 K directly; others require BF16 K)."
)
if not case.forward_mode.is_extend_without_speculative():
raise ValueError("run_dsa_sparse_fp8_prefill_case expects an EXTEND case.")