From 8b5d77c2688ef3108efc1ef52b99c352e5b52c7a Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Thu, 24 Sep 2026 09:59:17 +0800 Subject: [PATCH] [AMD] Add a Triton packed sparse decode path for QSA on ROCm (#38876) --- .../srt/layers/attention/qsa/sparse_attn.py | 53 +++++++++++++++++++ .../attention/qwen_sparse_attn_backend.py | 23 +++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/attention/qsa/sparse_attn.py b/python/sglang/srt/layers/attention/qsa/sparse_attn.py index 601945f053..a52c1e510a 100644 --- a/python/sglang/srt/layers/attention/qsa/sparse_attn.py +++ b/python/sglang/srt/layers/attention/qsa/sparse_attn.py @@ -316,6 +316,58 @@ def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, sc return out +def sparse_gqa_packed_decode_triton(q, k, v, indices, cu_q, cu_k, kv_lens, scale): + """Run one packed sparse-attention row per request without a host sync. + + Reuses the chunk-prefill kernel at a fixed query length of one, so graph + capture never hits the ``.item()`` that the general interface needs to + derive ``max_q``. + """ + + k, v = k.contiguous(), v.contiguous() + total_q, num_q_heads, head_dim = q.shape + num_kv_heads = k.shape[1] + group_size = num_q_heads // num_kv_heads + block_m = max(16, triton.next_power_of_2(group_size)) + block_n, warps, stages = _get_best_config(total_q) + out = torch.empty_like(q) + _sparse_gqa_chunk_prefill[(1, (cu_q.shape[0] - 1) * num_kv_heads)]( + q, + k, + v, + out, + indices, + cu_q, + cu_k, + kv_lens, + scale, + indices.shape[-1], + q.stride(0), + q.stride(1), + q.stride(2), + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + out.stride(0), + out.stride(1), + out.stride(2), + indices.stride(0), + indices.stride(1) if indices.ndim == 3 else 0, + indices.stride(2) if indices.ndim == 3 else indices.stride(1), + NUM_KV_HEADS=num_kv_heads, + GROUP_SIZE=group_size, + BLOCK_M=block_m, + BLOCK_N=block_n, + HEAD_DIM=head_dim, + num_warps=warps, + num_stages=stages, + ) + return out + + @triton.jit def _fa2_valid_counts( seq_lens, @@ -518,4 +570,5 @@ __all__ = [ "qwen_sparse_kv_extraction_compact_triton", "sparse_gqa_fwd_interface_triton", "sparse_gqa_fwd_interface_triton_ck", + "sparse_gqa_packed_decode_triton", ] diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index 9cd60707f9..b6b032dc06 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -35,8 +35,10 @@ from sglang.srt.layers.attention.qsa.sparse_attn import ( qwen_sparse_valid_counts_triton, sparse_gqa_fwd_interface_triton, sparse_gqa_fwd_interface_triton_ck, + sparse_gqa_packed_decode_triton, ) from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.utils import is_hip logger = logging.getLogger(__name__) @@ -1547,7 +1549,6 @@ class QwenSparseAttnBackend(AttentionBackend): trtllm_decode, ) - flash_attn_varlen_func = _resolve_flash_attn_varlen_func() batch, topk = topk_indices.shape sequence_lens = metadata.sequence_lengths if metadata.is_cuda_graph: @@ -1597,6 +1598,26 @@ class QwenSparseAttnBackend(AttentionBackend): batch, topk, ) + if is_hip(): + relative_indices = torch.arange( + topk, dtype=torch.int32, device=q.device + ).expand(batch, -1) + relative_indices = relative_indices.masked_fill( + relative_indices >= valid_counts[:, None], -1 + ).contiguous() + output = sparse_gqa_packed_decode_triton( + q.contiguous(), + packed_k, + packed_v, + relative_indices, + cu_seqlens_q, + cu_seqlens_k, + valid_counts, + layer.scaling, + ) + return output.reshape(q.shape[0], -1) + + flash_attn_varlen_func = _resolve_flash_attn_varlen_func() output = flash_attn_varlen_func( q=q, k=packed_k,