[AMD] Add a Triton packed sparse decode path for QSA on ROCm (#38876)
This commit is contained in:
@@ -316,6 +316,58 @@ def sparse_gqa_fwd_interface_triton_ck(q, k, v, indices, cu_q, cu_k, kv_lens, sc
|
||||
return out
|
||||
|
||||
|
||||
def sparse_gqa_packed_decode_triton(q, k, v, indices, cu_q, cu_k, kv_lens, scale):
|
||||
"""Run one packed sparse-attention row per request without a host sync.
|
||||
|
||||
Reuses the chunk-prefill kernel at a fixed query length of one, so graph
|
||||
capture never hits the ``.item()`` that the general interface needs to
|
||||
derive ``max_q``.
|
||||
"""
|
||||
|
||||
k, v = k.contiguous(), v.contiguous()
|
||||
total_q, num_q_heads, head_dim = q.shape
|
||||
num_kv_heads = k.shape[1]
|
||||
group_size = num_q_heads // num_kv_heads
|
||||
block_m = max(16, triton.next_power_of_2(group_size))
|
||||
block_n, warps, stages = _get_best_config(total_q)
|
||||
out = torch.empty_like(q)
|
||||
_sparse_gqa_chunk_prefill[(1, (cu_q.shape[0] - 1) * num_kv_heads)](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
out,
|
||||
indices,
|
||||
cu_q,
|
||||
cu_k,
|
||||
kv_lens,
|
||||
scale,
|
||||
indices.shape[-1],
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
k.stride(0),
|
||||
k.stride(1),
|
||||
k.stride(2),
|
||||
v.stride(0),
|
||||
v.stride(1),
|
||||
v.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
indices.stride(0),
|
||||
indices.stride(1) if indices.ndim == 3 else 0,
|
||||
indices.stride(2) if indices.ndim == 3 else indices.stride(1),
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
GROUP_SIZE=group_size,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
HEAD_DIM=head_dim,
|
||||
num_warps=warps,
|
||||
num_stages=stages,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fa2_valid_counts(
|
||||
seq_lens,
|
||||
@@ -518,4 +570,5 @@ __all__ = [
|
||||
"qwen_sparse_kv_extraction_compact_triton",
|
||||
"sparse_gqa_fwd_interface_triton",
|
||||
"sparse_gqa_fwd_interface_triton_ck",
|
||||
"sparse_gqa_packed_decode_triton",
|
||||
]
|
||||
|
||||
@@ -35,8 +35,10 @@ from sglang.srt.layers.attention.qsa.sparse_attn import (
|
||||
qwen_sparse_valid_counts_triton,
|
||||
sparse_gqa_fwd_interface_triton,
|
||||
sparse_gqa_fwd_interface_triton_ck,
|
||||
sparse_gqa_packed_decode_triton,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1547,7 +1549,6 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
trtllm_decode,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = _resolve_flash_attn_varlen_func()
|
||||
batch, topk = topk_indices.shape
|
||||
sequence_lens = metadata.sequence_lengths
|
||||
if metadata.is_cuda_graph:
|
||||
@@ -1597,6 +1598,26 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
batch,
|
||||
topk,
|
||||
)
|
||||
if is_hip():
|
||||
relative_indices = torch.arange(
|
||||
topk, dtype=torch.int32, device=q.device
|
||||
).expand(batch, -1)
|
||||
relative_indices = relative_indices.masked_fill(
|
||||
relative_indices >= valid_counts[:, None], -1
|
||||
).contiguous()
|
||||
output = sparse_gqa_packed_decode_triton(
|
||||
q.contiguous(),
|
||||
packed_k,
|
||||
packed_v,
|
||||
relative_indices,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_k,
|
||||
valid_counts,
|
||||
layer.scaling,
|
||||
)
|
||||
return output.reshape(q.shape[0], -1)
|
||||
|
||||
flash_attn_varlen_func = _resolve_flash_attn_varlen_func()
|
||||
output = flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=packed_k,
|
||||
|
||||
Reference in New Issue
Block a user