[Perf] [AMD] attention: Spread uneven Triton decode batches across XCDs (#41229)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Siliang Chen
2026-10-19 23:03:27 +08:00
committed by GitHub
co-authored by Cursor
parent 96ba108072
commit 7c4cc5a51c
4 changed files with 166 additions and 7 deletions
@@ -45,6 +45,15 @@ logger = logging.getLogger(__name__)
_MIN_BLOCK_KV = 32
def _rotate_batch(batch: int, uneven_batch: Optional[torch.Tensor]) -> bool:
# HIP hands workgroups to XCDs round-robin. With batch as the fastest grid axis,
# an even batch keeps each request on a subset of XCDs, so a long request or
# CUDA-graph padding rows leave the other XCDs idle. A balanced batch is already
# spread evenly, so the kernels only rotate while uneven_batch holds 1.
return _is_hip and batch % 2 == 0 and uneven_batch is not None
# heads per stage-1 tile, shared so the budget's head_tiles cannot drift from the launch
_GROUPED_BLOCK_H = 16
@@ -264,12 +273,18 @@ def _fwd_kernel_stage1(
aux0_stride_t=0,
aux0_stride_h=0,
aux0_len=0,
uneven_batch=None,
ROTATE_BATCH: tl.constexpr = False,
):
# int64 to avoid overflow of flat offsets into Mid_O when
# batch * num_head * max_kv_splits * head_dim exceeds 2**31.
cur_batch = tl.program_id(0).to(tl.int64)
cur_batch = tl.program_id(0)
cur_head = tl.program_id(1)
split_kv_id = tl.program_id(2)
if ROTATE_BATCH:
if tl.load(uneven_batch) != 0:
cur_batch = (cur_batch + cur_head + split_kv_id) % tl.num_programs(0)
# int64 to avoid overflow of flat offsets into Mid_O when
# batch * num_head * max_kv_splits * head_dim exceeds 2**31.
cur_batch = cur_batch.to(tl.int64)
cur_kv_head = cur_head // kv_group_num
@@ -428,6 +443,7 @@ def _decode_att_m_fwd(
page_size: int = 1,
score_mod=None,
aux_tensors=None,
uneven_batch=None,
):
BLOCK = 64
# [TODO] work around SGPR limit on MI3xx
@@ -505,6 +521,8 @@ def _decode_att_m_fwd(
aux0_stride_t=aux0_stride_t,
aux0_stride_h=aux0_stride_h,
aux0_len=aux0_len,
uneven_batch=uneven_batch,
ROTATE_BATCH=_rotate_batch(batch, uneven_batch),
)
@@ -556,13 +574,21 @@ def _fwd_grouped_kernel_stage1(
aux0_len=0,
forced_kv_splits=0,
USE_FORCED: tl.constexpr = False,
uneven_batch=None,
ROTATE_BATCH: tl.constexpr = False,
):
cur_batch = tl.program_id(0)
cur_head_id = tl.program_id(1)
split_kv_id = tl.program_id(2)
if ROTATE_BATCH:
# Only the split rotates: the head tiles of one (request, split) read the same
# kv_indices (and the same latent KV under MLA), so they stay on one XCD.
if tl.load(uneven_batch) != 0:
cur_batch = (cur_batch + split_kv_id) % tl.num_programs(0)
# int64 to avoid overflow of flat offsets into Mid_O when
# batch * num_head * max_kv_splits * head_dim exceeds 2**31.
cur_batch = tl.program_id(0).to(tl.int64)
cur_head_id = tl.program_id(1)
cur_batch = cur_batch.to(tl.int64)
cur_kv_head = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H)
split_kv_id = tl.program_id(2)
if BLOCK_H < kv_group_num:
VALID_BLOCK_H: tl.constexpr = BLOCK_H
@@ -787,6 +813,7 @@ def _decode_grouped_att_m_fwd(
aux_tensors=None,
tune_mla: bool = False,
forced_kv_splits: int = 0,
uneven_batch=None,
):
BLOCK = 32
Lk = k_buffer.shape[-1]
@@ -898,6 +925,8 @@ def _decode_grouped_att_m_fwd(
aux0_len=aux0_len,
forced_kv_splits=forced_kv_splits,
USE_FORCED=forced_kv_splits > 0,
uneven_batch=uneven_batch,
ROTATE_BATCH=_rotate_batch(batch, uneven_batch),
**extra_kargs,
)
@@ -1063,6 +1092,7 @@ def decode_attention_fwd_normal(
page_size: int = 1,
score_mod=None,
aux_tensors=None,
uneven_batch=None,
):
_decode_att_m_fwd(
q,
@@ -1080,6 +1110,7 @@ def decode_attention_fwd_normal(
page_size=page_size,
score_mod=score_mod,
aux_tensors=aux_tensors,
uneven_batch=uneven_batch,
)
_decode_softmax_reducev_fwd(
attn_logits,
@@ -1116,6 +1147,7 @@ def decode_attention_fwd_grouped(
page_size: int = 1,
score_mod=None,
aux_tensors=None,
uneven_batch=None,
):
tune_mla, forced_kv_splits = _mla_launch_plan(q, k_buffer, max_kv_splits, has_mla)
_decode_grouped_att_m_fwd(
@@ -1138,6 +1170,7 @@ def decode_attention_fwd_grouped(
aux_tensors=aux_tensors,
tune_mla=tune_mla,
forced_kv_splits=forced_kv_splits,
uneven_batch=uneven_batch,
)
_decode_softmax_reducev_fwd(
attn_logits,
@@ -1182,6 +1215,7 @@ def decode_attention_fwd(
lean_Lp=None,
lean_Op=None,
lean_locks=None,
uneven_batch=None,
):
assert max_kv_splits == attn_logits.shape[2]
assert q.shape[0] <= kv_indptr.shape[0] - 1
@@ -1253,6 +1287,7 @@ def decode_attention_fwd(
page_size=page_size,
score_mod=score_mod,
aux_tensors=aux_tensors,
uneven_batch=uneven_batch,
)
else:
# GQA/MQA/MLA
@@ -1277,6 +1312,7 @@ def decode_attention_fwd(
page_size=page_size,
score_mod=score_mod,
aux_tensors=aux_tensors,
uneven_batch=uneven_batch,
)
@@ -16,6 +16,7 @@ def get_num_kv_splits_triton(
max_kv_splits,
device_core_count,
MAX_NUM_SEQ: tl.constexpr,
uneven_batch_ptr=None,
):
# TODO: this method is tunable, we need more online serving data to tune it
offs_seq = tl.arange(0, MAX_NUM_SEQ)
@@ -27,6 +28,8 @@ def get_num_kv_splits_triton(
min_seq_len = tl.min(seq_lens)
if max_seq_len * 8 < min_seq_len * 10:
min_seq_len = max_seq_len
if uneven_batch_ptr is not None:
tl.store(uneven_batch_ptr, (min_seq_len < max_seq_len).to(tl.int32))
max_kv_splits_1 = tl.minimum(tl.cdiv(max_seq_len, min_seq_len), max_kv_splits)
kv_chunk_size_1 = tl.cdiv(max_seq_len, max_kv_splits_1)
@@ -143,6 +143,8 @@ class ForwardMetadata:
lean_Lp: Optional[torch.Tensor] = None
lean_Op: Optional[torch.Tensor] = None
lean_locks: Optional[torch.Tensor] = None
# One int32, set by get_num_kv_splits when the decode seq lens are uneven
uneven_batch: Optional[torch.Tensor] = None
class TritonAttnBackend(AttentionBackend):
@@ -430,6 +432,7 @@ class TritonAttnBackend(AttentionBackend):
self,
num_kv_splits: torch.Tensor,
seq_lens: torch.Tensor,
uneven_batch: Optional[torch.Tensor] = None,
):
num_token, num_seq = num_kv_splits.shape[0], seq_lens.shape[0]
# NOTE(alcanderian): Considering speculative_decodeing,
@@ -473,6 +476,7 @@ class TritonAttnBackend(AttentionBackend):
self.max_kv_splits,
self.device_core_count,
MAX_NUM_SEQ=SCHEDULE_SEQ,
uneven_batch_ptr=uneven_batch,
)
def _dcp_lens(self, lens: torch.Tensor, start: Optional[torch.Tensor] = None):
@@ -881,6 +885,7 @@ class TritonAttnBackend(AttentionBackend):
# Lean decode buffers are only allocated on the decode path below; default
# to None so the shared ForwardMetadata constructor works for extend/verify.
lean_Mp = lean_Lp = lean_Op = lean_locks = None
uneven_batch = None
if forward_batch.forward_mode.is_decode_or_idle():
if spec_info is None or spec_info.kv_indptr is None:
@@ -964,6 +969,7 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
num_kv_splits = torch.empty((bs,), dtype=torch.int32, device=self.device)
uneven_batch = torch.zeros((1,), dtype=torch.int32, device=self.device)
self.get_num_kv_splits(
num_kv_splits,
(
@@ -971,6 +977,7 @@ class TritonAttnBackend(AttentionBackend):
if self.dcp_size > 1
else forward_batch.seq_lens
),
uneven_batch,
)
# Lean decode persistent-grid partial-result buffers.
@@ -1147,6 +1154,7 @@ class TritonAttnBackend(AttentionBackend):
lean_Lp=lean_Lp,
lean_Op=lean_Op,
lean_locks=lean_locks,
uneven_batch=uneven_batch,
)
def init_cuda_graph_state(
@@ -1211,6 +1219,9 @@ class TritonAttnBackend(AttentionBackend):
)
else:
self.cuda_graph_num_kv_splits = cuda_graph_num_kv_splits_buf
self.cuda_graph_uneven_batch = torch.zeros(
(1,), dtype=torch.int32, device=self.device
)
if kv_indices_buf is None:
self.cuda_graph_kv_indices = torch.zeros(
@@ -1305,6 +1316,7 @@ class TritonAttnBackend(AttentionBackend):
lean_Lp=self.cuda_graph_lean_Lp,
lean_Op=self.cuda_graph_lean_Op,
lean_locks=self.cuda_graph_lean_locks,
uneven_batch=self.cuda_graph_uneven_batch,
)
elif forward_mode.is_target_verify():
custom_mask = (
@@ -1401,7 +1413,9 @@ class TritonAttnBackend(AttentionBackend):
plan, bs, seq_lens, req_pool_indices
)
self.get_num_kv_splits(
self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs]
self.cuda_graph_num_kv_splits[:bs],
num_kv_splits_lens[:bs],
self.cuda_graph_uneven_batch,
)
if window_kv_lens is not None:
self.get_num_kv_splits(
@@ -2444,6 +2458,7 @@ class TritonAttnBackend(AttentionBackend):
lean_Lp=self.forward_metadata.lean_Lp,
lean_Op=self.forward_metadata.lean_Op,
lean_locks=self.forward_metadata.lean_locks,
uneven_batch=self.forward_metadata.uneven_batch,
)
local_lse = torch.logsumexp(
self.forward_metadata.attn_lse[
@@ -2481,6 +2496,7 @@ class TritonAttnBackend(AttentionBackend):
lean_Lp=self.forward_metadata.lean_Lp,
lean_Op=self.forward_metadata.lean_Op,
lean_locks=self.forward_metadata.lean_locks,
uneven_batch=self.forward_metadata.uneven_batch,
)
return o