From 0931dfedc9614e1369834ec1e356ec011afa7b27 Mon Sep 17 00:00:00 2001 From: Pawel Kowalski Date: Fri, 23 Oct 2026 21:30:42 +0200 Subject: [PATCH] [Fix] [AMD] dsa: Fix AITER prefill with an FP8 KV cache (#39083) Co-authored-by: Pawel Kowalski Co-authored-by: Thomas Wang --- .../srt/layers/attention/dsa_backend.py | 50 +++++++++++++------ .../attention_methods/dsa_attention.py | 6 +-- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 1ff1b028bd..bcd8173c5e 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -644,6 +644,9 @@ class DeepseekSparseAttnBackend( self.aiter_dsa_metadata_kv_dtype = None self.aiter_dsa_kv_last_page_lens = None self.aiter_dsa_work_metadata = None + self.aiter_dsa_identity_scale = torch.ones( + (), dtype=torch.float32, device=self.device + ) if ( self.dsa_prefill_impl == "aiter" or self.dsa_decode_impl == "aiter" @@ -802,9 +805,8 @@ class DeepseekSparseAttnBackend( q_dtype, kv_dtype, is_sparse=True, - fast_mode=False, - num_kv_splits=self.aiter_dsa_max_split_per_batch, - intra_batch_mode=True, + fast_mode=True, + intra_batch_mode=False, ) return ( @@ -900,10 +902,10 @@ class DeepseekSparseAttnBackend( kv_granularity=16, max_seqlen_qo=max_seqlen_q, uni_seqlen_qo=max_seqlen_q, - fast_mode=False, - topk=self.dsa_index_topk, + fast_mode=True, + topk=-1, max_split_per_batch=self.aiter_dsa_max_split_per_batch, - intra_batch_mode=True, + intra_batch_mode=False, dtype_q=q_dtype, dtype_kv=kv_dtype, ) @@ -916,8 +918,6 @@ class DeepseekSparseAttnBackend( "reduce_indptr": self.aiter_dsa_reduce_indptr, "reduce_final_map": self.aiter_dsa_reduce_final_map, "reduce_partial_map": self.aiter_dsa_reduce_partial_map, - "intra_batch_mode": True, - "num_kv_splits": self.aiter_dsa_max_split_per_batch, } def _pad_trtllm_sparse_page_table( @@ -3620,11 +3620,14 @@ class DeepseekSparseAttnBackend( bs: int, ) -> torch.Tensor: q = q_all.reshape(-1, layer.tp_q_head_num * layer.head_dim) + o_dtype = torch.bfloat16 if q.dtype == fp8_dtype else q.dtype if layer.head_dim != layer.v_head_dim: - o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + o = q.new_empty( + (q.shape[0], layer.tp_q_head_num * layer.v_head_dim), dtype=o_dtype + ) else: - o = torch.empty_like(q) + o = torch.empty_like(q, dtype=o_dtype) if self.need_pad_heads: q_kernel = q.view( @@ -3635,7 +3638,8 @@ class DeepseekSparseAttnBackend( q.shape[0], layer.tp_q_head_num * self.head_repeat_factor, layer.v_head_dim, - ) + ), + dtype=o_dtype, ) else: q_kernel = q.view(-1, layer.tp_q_head_num, layer.head_dim) @@ -3645,7 +3649,12 @@ class DeepseekSparseAttnBackend( kv_scale = None aiter_persistent_kwargs = {} if kv_cache.dtype == fp8_dtype: - kv_scale = torch.ones((), dtype=torch.float32, device=q_kernel.device) + q_scale = self.aiter_dsa_identity_scale + kv_scale = ( + layer.k_scale.reshape(()) + if isinstance(layer.k_scale, torch.Tensor) + else self.aiter_dsa_identity_scale + ) kv_indptr = self.kv_indptr @@ -3698,11 +3707,14 @@ class DeepseekSparseAttnBackend( ) -> torch.Tensor: num_tokens = q_all.shape[0] q = q_all.reshape(-1, layer.tp_q_head_num * layer.head_dim) + o_dtype = torch.bfloat16 if q.dtype == fp8_dtype else q.dtype if layer.head_dim != layer.v_head_dim: - o = q.new_empty((num_tokens, layer.tp_q_head_num * layer.v_head_dim)) + o = q.new_empty( + (num_tokens, layer.tp_q_head_num * layer.v_head_dim), dtype=o_dtype + ) else: - o = torch.empty_like(q) + o = torch.empty_like(q, dtype=o_dtype) if self.need_pad_heads: q_kernel = q.view( @@ -3713,7 +3725,8 @@ class DeepseekSparseAttnBackend( num_tokens, layer.tp_q_head_num * self.head_repeat_factor, layer.v_head_dim, - ) + ), + dtype=o_dtype, ) else: q_kernel = q.view(-1, layer.tp_q_head_num, layer.head_dim) @@ -3723,7 +3736,12 @@ class DeepseekSparseAttnBackend( kv_scale = None aiter_persistent_kwargs = {} if kv_cache.dtype == fp8_dtype: - kv_scale = torch.ones((), dtype=torch.float32, device=q_kernel.device) + q_scale = self.aiter_dsa_identity_scale + kv_scale = ( + layer.k_scale.reshape(()) + if isinstance(layer.k_scale, torch.Tensor) + else self.aiter_dsa_identity_scale + ) non_minus1_mask = page_table_1 != -1 non_minus1_counts = non_minus1_mask.sum(dim=1) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index dd77de6d80..12d91a1310 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -1248,7 +1248,7 @@ DSA_DECODE_IMPL_VARIANTS: tuple[str, ...] = ( # take in FP8 deployments. The `flashmla_kv` decode kernel and *both* # flashmla prefill kernels are the production-relevant FP8 paths. DSA_FP8_COMPATIBLE_PREFILL_IMPLS: frozenset[str] = frozenset( - {"flashmla_sparse", "flashmla_kv", "flashmla_auto", "triton_sparse_mla"} + {"aiter", "flashmla_sparse", "flashmla_kv", "flashmla_auto", "triton_sparse_mla"} ) DSA_FP8_COMPATIBLE_DECODE_IMPLS: frozenset[str] = frozenset( {"flashmla_kv", "flashmla_auto"} @@ -1395,8 +1395,8 @@ def run_dsa_sparse_fp8_prefill_case( if dsa_prefill_backend not in DSA_FP8_COMPATIBLE_PREFILL_IMPLS: testcase.skipTest( f"DSA prefill impl `{dsa_prefill_backend}` does not support FP8 KV " - f"cache (only `flashmla_sparse`, `flashmla_kv`, and `flashmla_auto` " - f"read FP8 K directly; others require BF16 K)." + f"cache (only `aiter`, `flashmla_sparse`, `flashmla_kv`, and " + f"`flashmla_auto` read FP8 K directly; others require BF16 K)." ) if not case.forward_mode.is_extend_without_speculative(): raise ValueError("run_dsa_sparse_fp8_prefill_case expects an EXTEND case.")