[AMD] Add a Triton packed sparse decode path for QSA on ROCm (#38876)

This commit is contained in:
jacky.cheng
2026-09-23 18:59:17 -07:00
committed by GitHub
parent 82cdd72588
commit 8b5d77c268
2 changed files with 75 additions and 1 deletions
@@ -316,6 +316,58 @@ def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, sc
return out
def sparse_gqa_packed_decode_triton(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
"""Run one packed sparse-attention row per request without a host sync.
Reuses the chunk-prefill kernel at a fixed query length of one, so graph
capture never hits the ``.item()`` that the general interface needs to
derive ``max_q``.
"""
k, v = k.contiguous(), v.contiguous()
total_q, num_q_heads, head_dim = q.shape
num_kv_heads = k.shape[1]
group_size = num_q_heads // num_kv_heads
block_m = max(16, triton.next_power_of_2(group_size))
block_n, warps, stages = _get_best_config(total_q)
out = torch.empty_like(q)
_sparse_gqa_chunk_prefill[(1, (cu_q.shape[0] - 1) * num_kv_heads)](
q,
k,
v,
out,
indices,
cu_q,
cu_k,
kv_lens,
scale,
indices.shape[-1],
q.stride(0),
q.stride(1),
q.stride(2),
k.stride(0),
k.stride(1),
k.stride(2),
v.stride(0),
v.stride(1),
v.stride(2),
out.stride(0),
out.stride(1),
out.stride(2),
indices.stride(0),
indices.stride(1) if indices.ndim == 3 else 0,
indices.stride(2) if indices.ndim == 3 else indices.stride(1),
NUM_KV_HEADS=num_kv_heads,
GROUP_SIZE=group_size,
BLOCK_M=block_m,
BLOCK_N=block_n,
HEAD_DIM=head_dim,
num_warps=warps,
num_stages=stages,
)
return out
@triton.jit
def _fa2_valid_counts(
seq_lens,
@@ -518,4 +570,5 @@ __all__ = [
"qwen_sparse_kv_extraction_compact_triton",
"sparse_gqa_fwd_interface_triton",
"sparse_gqa_fwd_interface_triton_ck",
"sparse_gqa_packed_decode_triton",
]
@@ -35,8 +35,10 @@ from sglang.srt.layers.attention.qsa.sparse_attn import (
qwen_sparse_valid_counts_triton,
sparse_gqa_fwd_interface_triton,
sparse_gqa_fwd_interface_triton_ck,
sparse_gqa_packed_decode_triton,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.utils import is_hip
logger = logging.getLogger(__name__)
@@ -1547,7 +1549,6 @@ class QwenSparseAttnBackend(AttentionBackend):
trtllm_decode,
)
flash_attn_varlen_func = _resolve_flash_attn_varlen_func()
batch, topk = topk_indices.shape
sequence_lens = metadata.sequence_lengths
if metadata.is_cuda_graph:
@@ -1597,6 +1598,26 @@ class QwenSparseAttnBackend(AttentionBackend):
batch,
topk,
)
if is_hip():
relative_indices = torch.arange(
topk, dtype=torch.int32, device=q.device
).expand(batch, -1)
relative_indices = relative_indices.masked_fill(
relative_indices >= valid_counts[:, None], -1
).contiguous()
output = sparse_gqa_packed_decode_triton(
q.contiguous(),
packed_k,
packed_v,
relative_indices,
cu_seqlens_q,
cu_seqlens_k,
valid_counts,
layer.scaling,
)
return output.reshape(q.shape[0], -1)
flash_attn_varlen_func = _resolve_flash_attn_varlen_func()
output = flash_attn_varlen_func(
q=q,
k=packed_k,