[Perf] [AMD] attention: Spread uneven Triton decode batches across XCDs (#41229)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -45,6 +45,15 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_MIN_BLOCK_KV = 32
|
||||
|
||||
|
||||
def _rotate_batch(batch: int, uneven_batch: Optional[torch.Tensor]) -> bool:
|
||||
# HIP hands workgroups to XCDs round-robin. With batch as the fastest grid axis,
|
||||
# an even batch keeps each request on a subset of XCDs, so a long request or
|
||||
# CUDA-graph padding rows leave the other XCDs idle. A balanced batch is already
|
||||
# spread evenly, so the kernels only rotate while uneven_batch holds 1.
|
||||
return _is_hip and batch % 2 == 0 and uneven_batch is not None
|
||||
|
||||
|
||||
# heads per stage-1 tile, shared so the budget's head_tiles cannot drift from the launch
|
||||
_GROUPED_BLOCK_H = 16
|
||||
|
||||
@@ -264,12 +273,18 @@ def _fwd_kernel_stage1(
|
||||
aux0_stride_t=0,
|
||||
aux0_stride_h=0,
|
||||
aux0_len=0,
|
||||
uneven_batch=None,
|
||||
ROTATE_BATCH: tl.constexpr = False,
|
||||
):
|
||||
# int64 to avoid overflow of flat offsets into Mid_O when
|
||||
# batch * num_head * max_kv_splits * head_dim exceeds 2**31.
|
||||
cur_batch = tl.program_id(0).to(tl.int64)
|
||||
cur_batch = tl.program_id(0)
|
||||
cur_head = tl.program_id(1)
|
||||
split_kv_id = tl.program_id(2)
|
||||
if ROTATE_BATCH:
|
||||
if tl.load(uneven_batch) != 0:
|
||||
cur_batch = (cur_batch + cur_head + split_kv_id) % tl.num_programs(0)
|
||||
# int64 to avoid overflow of flat offsets into Mid_O when
|
||||
# batch * num_head * max_kv_splits * head_dim exceeds 2**31.
|
||||
cur_batch = cur_batch.to(tl.int64)
|
||||
|
||||
cur_kv_head = cur_head // kv_group_num
|
||||
|
||||
@@ -428,6 +443,7 @@ def _decode_att_m_fwd(
|
||||
page_size: int = 1,
|
||||
score_mod=None,
|
||||
aux_tensors=None,
|
||||
uneven_batch=None,
|
||||
):
|
||||
BLOCK = 64
|
||||
# [TODO] work around SGPR limit on MI3xx
|
||||
@@ -505,6 +521,8 @@ def _decode_att_m_fwd(
|
||||
aux0_stride_t=aux0_stride_t,
|
||||
aux0_stride_h=aux0_stride_h,
|
||||
aux0_len=aux0_len,
|
||||
uneven_batch=uneven_batch,
|
||||
ROTATE_BATCH=_rotate_batch(batch, uneven_batch),
|
||||
)
|
||||
|
||||
|
||||
@@ -556,13 +574,21 @@ def _fwd_grouped_kernel_stage1(
|
||||
aux0_len=0,
|
||||
forced_kv_splits=0,
|
||||
USE_FORCED: tl.constexpr = False,
|
||||
uneven_batch=None,
|
||||
ROTATE_BATCH: tl.constexpr = False,
|
||||
):
|
||||
cur_batch = tl.program_id(0)
|
||||
cur_head_id = tl.program_id(1)
|
||||
split_kv_id = tl.program_id(2)
|
||||
if ROTATE_BATCH:
|
||||
# Only the split rotates: the head tiles of one (request, split) read the same
|
||||
# kv_indices (and the same latent KV under MLA), so they stay on one XCD.
|
||||
if tl.load(uneven_batch) != 0:
|
||||
cur_batch = (cur_batch + split_kv_id) % tl.num_programs(0)
|
||||
# int64 to avoid overflow of flat offsets into Mid_O when
|
||||
# batch * num_head * max_kv_splits * head_dim exceeds 2**31.
|
||||
cur_batch = tl.program_id(0).to(tl.int64)
|
||||
cur_head_id = tl.program_id(1)
|
||||
cur_batch = cur_batch.to(tl.int64)
|
||||
cur_kv_head = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H)
|
||||
split_kv_id = tl.program_id(2)
|
||||
|
||||
if BLOCK_H < kv_group_num:
|
||||
VALID_BLOCK_H: tl.constexpr = BLOCK_H
|
||||
@@ -787,6 +813,7 @@ def _decode_grouped_att_m_fwd(
|
||||
aux_tensors=None,
|
||||
tune_mla: bool = False,
|
||||
forced_kv_splits: int = 0,
|
||||
uneven_batch=None,
|
||||
):
|
||||
BLOCK = 32
|
||||
Lk = k_buffer.shape[-1]
|
||||
@@ -898,6 +925,8 @@ def _decode_grouped_att_m_fwd(
|
||||
aux0_len=aux0_len,
|
||||
forced_kv_splits=forced_kv_splits,
|
||||
USE_FORCED=forced_kv_splits > 0,
|
||||
uneven_batch=uneven_batch,
|
||||
ROTATE_BATCH=_rotate_batch(batch, uneven_batch),
|
||||
**extra_kargs,
|
||||
)
|
||||
|
||||
@@ -1063,6 +1092,7 @@ def decode_attention_fwd_normal(
|
||||
page_size: int = 1,
|
||||
score_mod=None,
|
||||
aux_tensors=None,
|
||||
uneven_batch=None,
|
||||
):
|
||||
_decode_att_m_fwd(
|
||||
q,
|
||||
@@ -1080,6 +1110,7 @@ def decode_attention_fwd_normal(
|
||||
page_size=page_size,
|
||||
score_mod=score_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
uneven_batch=uneven_batch,
|
||||
)
|
||||
_decode_softmax_reducev_fwd(
|
||||
attn_logits,
|
||||
@@ -1116,6 +1147,7 @@ def decode_attention_fwd_grouped(
|
||||
page_size: int = 1,
|
||||
score_mod=None,
|
||||
aux_tensors=None,
|
||||
uneven_batch=None,
|
||||
):
|
||||
tune_mla, forced_kv_splits = _mla_launch_plan(q, k_buffer, max_kv_splits, has_mla)
|
||||
_decode_grouped_att_m_fwd(
|
||||
@@ -1138,6 +1170,7 @@ def decode_attention_fwd_grouped(
|
||||
aux_tensors=aux_tensors,
|
||||
tune_mla=tune_mla,
|
||||
forced_kv_splits=forced_kv_splits,
|
||||
uneven_batch=uneven_batch,
|
||||
)
|
||||
_decode_softmax_reducev_fwd(
|
||||
attn_logits,
|
||||
@@ -1182,6 +1215,7 @@ def decode_attention_fwd(
|
||||
lean_Lp=None,
|
||||
lean_Op=None,
|
||||
lean_locks=None,
|
||||
uneven_batch=None,
|
||||
):
|
||||
assert max_kv_splits == attn_logits.shape[2]
|
||||
assert q.shape[0] <= kv_indptr.shape[0] - 1
|
||||
@@ -1253,6 +1287,7 @@ def decode_attention_fwd(
|
||||
page_size=page_size,
|
||||
score_mod=score_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
uneven_batch=uneven_batch,
|
||||
)
|
||||
else:
|
||||
# GQA/MQA/MLA
|
||||
@@ -1277,6 +1312,7 @@ def decode_attention_fwd(
|
||||
page_size=page_size,
|
||||
score_mod=score_mod,
|
||||
aux_tensors=aux_tensors,
|
||||
uneven_batch=uneven_batch,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ def get_num_kv_splits_triton(
|
||||
max_kv_splits,
|
||||
device_core_count,
|
||||
MAX_NUM_SEQ: tl.constexpr,
|
||||
uneven_batch_ptr=None,
|
||||
):
|
||||
# TODO: this method is tunable, we need more online serving data to tune it
|
||||
offs_seq = tl.arange(0, MAX_NUM_SEQ)
|
||||
@@ -27,6 +28,8 @@ def get_num_kv_splits_triton(
|
||||
min_seq_len = tl.min(seq_lens)
|
||||
if max_seq_len * 8 < min_seq_len * 10:
|
||||
min_seq_len = max_seq_len
|
||||
if uneven_batch_ptr is not None:
|
||||
tl.store(uneven_batch_ptr, (min_seq_len < max_seq_len).to(tl.int32))
|
||||
max_kv_splits_1 = tl.minimum(tl.cdiv(max_seq_len, min_seq_len), max_kv_splits)
|
||||
kv_chunk_size_1 = tl.cdiv(max_seq_len, max_kv_splits_1)
|
||||
|
||||
|
||||
@@ -143,6 +143,8 @@ class ForwardMetadata:
|
||||
lean_Lp: Optional[torch.Tensor] = None
|
||||
lean_Op: Optional[torch.Tensor] = None
|
||||
lean_locks: Optional[torch.Tensor] = None
|
||||
# One int32, set by get_num_kv_splits when the decode seq lens are uneven
|
||||
uneven_batch: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
class TritonAttnBackend(AttentionBackend):
|
||||
@@ -430,6 +432,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self,
|
||||
num_kv_splits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
uneven_batch: Optional[torch.Tensor] = None,
|
||||
):
|
||||
num_token, num_seq = num_kv_splits.shape[0], seq_lens.shape[0]
|
||||
# NOTE(alcanderian): Considering speculative_decodeing,
|
||||
@@ -473,6 +476,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self.max_kv_splits,
|
||||
self.device_core_count,
|
||||
MAX_NUM_SEQ=SCHEDULE_SEQ,
|
||||
uneven_batch_ptr=uneven_batch,
|
||||
)
|
||||
|
||||
def _dcp_lens(self, lens: torch.Tensor, start: Optional[torch.Tensor] = None):
|
||||
@@ -881,6 +885,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
# Lean decode buffers are only allocated on the decode path below; default
|
||||
# to None so the shared ForwardMetadata constructor works for extend/verify.
|
||||
lean_Mp = lean_Lp = lean_Op = lean_locks = None
|
||||
uneven_batch = None
|
||||
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
if spec_info is None or spec_info.kv_indptr is None:
|
||||
@@ -964,6 +969,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
num_kv_splits = torch.empty((bs,), dtype=torch.int32, device=self.device)
|
||||
uneven_batch = torch.zeros((1,), dtype=torch.int32, device=self.device)
|
||||
self.get_num_kv_splits(
|
||||
num_kv_splits,
|
||||
(
|
||||
@@ -971,6 +977,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
if self.dcp_size > 1
|
||||
else forward_batch.seq_lens
|
||||
),
|
||||
uneven_batch,
|
||||
)
|
||||
|
||||
# Lean decode persistent-grid partial-result buffers.
|
||||
@@ -1147,6 +1154,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
lean_Lp=lean_Lp,
|
||||
lean_Op=lean_Op,
|
||||
lean_locks=lean_locks,
|
||||
uneven_batch=uneven_batch,
|
||||
)
|
||||
|
||||
def init_cuda_graph_state(
|
||||
@@ -1211,6 +1219,9 @@ class TritonAttnBackend(AttentionBackend):
|
||||
)
|
||||
else:
|
||||
self.cuda_graph_num_kv_splits = cuda_graph_num_kv_splits_buf
|
||||
self.cuda_graph_uneven_batch = torch.zeros(
|
||||
(1,), dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
if kv_indices_buf is None:
|
||||
self.cuda_graph_kv_indices = torch.zeros(
|
||||
@@ -1305,6 +1316,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
lean_Lp=self.cuda_graph_lean_Lp,
|
||||
lean_Op=self.cuda_graph_lean_Op,
|
||||
lean_locks=self.cuda_graph_lean_locks,
|
||||
uneven_batch=self.cuda_graph_uneven_batch,
|
||||
)
|
||||
elif forward_mode.is_target_verify():
|
||||
custom_mask = (
|
||||
@@ -1401,7 +1413,9 @@ class TritonAttnBackend(AttentionBackend):
|
||||
plan, bs, seq_lens, req_pool_indices
|
||||
)
|
||||
self.get_num_kv_splits(
|
||||
self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs]
|
||||
self.cuda_graph_num_kv_splits[:bs],
|
||||
num_kv_splits_lens[:bs],
|
||||
self.cuda_graph_uneven_batch,
|
||||
)
|
||||
if window_kv_lens is not None:
|
||||
self.get_num_kv_splits(
|
||||
@@ -2444,6 +2458,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
lean_Lp=self.forward_metadata.lean_Lp,
|
||||
lean_Op=self.forward_metadata.lean_Op,
|
||||
lean_locks=self.forward_metadata.lean_locks,
|
||||
uneven_batch=self.forward_metadata.uneven_batch,
|
||||
)
|
||||
local_lse = torch.logsumexp(
|
||||
self.forward_metadata.attn_lse[
|
||||
@@ -2481,6 +2496,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
lean_Lp=self.forward_metadata.lean_Lp,
|
||||
lean_Op=self.forward_metadata.lean_Op,
|
||||
lean_locks=self.forward_metadata.lean_locks,
|
||||
uneven_batch=self.forward_metadata.uneven_batch,
|
||||
)
|
||||
return o
|
||||
|
||||
|
||||
Reference in New Issue
Block a user