diff --git a/python/sglang/kernels/ops/attention/decode_attention.py b/python/sglang/kernels/ops/attention/decode_attention.py index 8bd69df94e..4c14429f2e 100644 --- a/python/sglang/kernels/ops/attention/decode_attention.py +++ b/python/sglang/kernels/ops/attention/decode_attention.py @@ -45,6 +45,15 @@ logger = logging.getLogger(__name__) _MIN_BLOCK_KV = 32 + +def _rotate_batch(batch: int, uneven_batch: Optional[torch.Tensor]) -> bool: + # HIP hands workgroups to XCDs round-robin. With batch as the fastest grid axis, + # an even batch keeps each request on a subset of XCDs, so a long request or + # CUDA-graph padding rows leave the other XCDs idle. A balanced batch is already + # spread evenly, so the kernels only rotate while uneven_batch holds 1. + return _is_hip and batch % 2 == 0 and uneven_batch is not None + + # heads per stage-1 tile, shared so the budget's head_tiles cannot drift from the launch _GROUPED_BLOCK_H = 16 @@ -264,12 +273,18 @@ def _fwd_kernel_stage1( aux0_stride_t=0, aux0_stride_h=0, aux0_len=0, + uneven_batch=None, + ROTATE_BATCH: tl.constexpr = False, ): - # int64 to avoid overflow of flat offsets into Mid_O when - # batch * num_head * max_kv_splits * head_dim exceeds 2**31. - cur_batch = tl.program_id(0).to(tl.int64) + cur_batch = tl.program_id(0) cur_head = tl.program_id(1) split_kv_id = tl.program_id(2) + if ROTATE_BATCH: + if tl.load(uneven_batch) != 0: + cur_batch = (cur_batch + cur_head + split_kv_id) % tl.num_programs(0) + # int64 to avoid overflow of flat offsets into Mid_O when + # batch * num_head * max_kv_splits * head_dim exceeds 2**31. + cur_batch = cur_batch.to(tl.int64) cur_kv_head = cur_head // kv_group_num @@ -428,6 +443,7 @@ def _decode_att_m_fwd( page_size: int = 1, score_mod=None, aux_tensors=None, + uneven_batch=None, ): BLOCK = 64 # [TODO] work around SGPR limit on MI3xx @@ -505,6 +521,8 @@ def _decode_att_m_fwd( aux0_stride_t=aux0_stride_t, aux0_stride_h=aux0_stride_h, aux0_len=aux0_len, + uneven_batch=uneven_batch, + ROTATE_BATCH=_rotate_batch(batch, uneven_batch), ) @@ -556,13 +574,21 @@ def _fwd_grouped_kernel_stage1( aux0_len=0, forced_kv_splits=0, USE_FORCED: tl.constexpr = False, + uneven_batch=None, + ROTATE_BATCH: tl.constexpr = False, ): + cur_batch = tl.program_id(0) + cur_head_id = tl.program_id(1) + split_kv_id = tl.program_id(2) + if ROTATE_BATCH: + # Only the split rotates: the head tiles of one (request, split) read the same + # kv_indices (and the same latent KV under MLA), so they stay on one XCD. + if tl.load(uneven_batch) != 0: + cur_batch = (cur_batch + split_kv_id) % tl.num_programs(0) # int64 to avoid overflow of flat offsets into Mid_O when # batch * num_head * max_kv_splits * head_dim exceeds 2**31. - cur_batch = tl.program_id(0).to(tl.int64) - cur_head_id = tl.program_id(1) + cur_batch = cur_batch.to(tl.int64) cur_kv_head = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H) - split_kv_id = tl.program_id(2) if BLOCK_H < kv_group_num: VALID_BLOCK_H: tl.constexpr = BLOCK_H @@ -787,6 +813,7 @@ def _decode_grouped_att_m_fwd( aux_tensors=None, tune_mla: bool = False, forced_kv_splits: int = 0, + uneven_batch=None, ): BLOCK = 32 Lk = k_buffer.shape[-1] @@ -898,6 +925,8 @@ def _decode_grouped_att_m_fwd( aux0_len=aux0_len, forced_kv_splits=forced_kv_splits, USE_FORCED=forced_kv_splits > 0, + uneven_batch=uneven_batch, + ROTATE_BATCH=_rotate_batch(batch, uneven_batch), **extra_kargs, ) @@ -1063,6 +1092,7 @@ def decode_attention_fwd_normal( page_size: int = 1, score_mod=None, aux_tensors=None, + uneven_batch=None, ): _decode_att_m_fwd( q, @@ -1080,6 +1110,7 @@ def decode_attention_fwd_normal( page_size=page_size, score_mod=score_mod, aux_tensors=aux_tensors, + uneven_batch=uneven_batch, ) _decode_softmax_reducev_fwd( attn_logits, @@ -1116,6 +1147,7 @@ def decode_attention_fwd_grouped( page_size: int = 1, score_mod=None, aux_tensors=None, + uneven_batch=None, ): tune_mla, forced_kv_splits = _mla_launch_plan(q, k_buffer, max_kv_splits, has_mla) _decode_grouped_att_m_fwd( @@ -1138,6 +1170,7 @@ def decode_attention_fwd_grouped( aux_tensors=aux_tensors, tune_mla=tune_mla, forced_kv_splits=forced_kv_splits, + uneven_batch=uneven_batch, ) _decode_softmax_reducev_fwd( attn_logits, @@ -1182,6 +1215,7 @@ def decode_attention_fwd( lean_Lp=None, lean_Op=None, lean_locks=None, + uneven_batch=None, ): assert max_kv_splits == attn_logits.shape[2] assert q.shape[0] <= kv_indptr.shape[0] - 1 @@ -1253,6 +1287,7 @@ def decode_attention_fwd( page_size=page_size, score_mod=score_mod, aux_tensors=aux_tensors, + uneven_batch=uneven_batch, ) else: # GQA/MQA/MLA @@ -1277,6 +1312,7 @@ def decode_attention_fwd( page_size=page_size, score_mod=score_mod, aux_tensors=aux_tensors, + uneven_batch=uneven_batch, ) diff --git a/python/sglang/kernels/ops/attention/metadata.py b/python/sglang/kernels/ops/attention/metadata.py index 3ded6383b1..2985807899 100644 --- a/python/sglang/kernels/ops/attention/metadata.py +++ b/python/sglang/kernels/ops/attention/metadata.py @@ -16,6 +16,7 @@ def get_num_kv_splits_triton( max_kv_splits, device_core_count, MAX_NUM_SEQ: tl.constexpr, + uneven_batch_ptr=None, ): # TODO: this method is tunable, we need more online serving data to tune it offs_seq = tl.arange(0, MAX_NUM_SEQ) @@ -27,6 +28,8 @@ def get_num_kv_splits_triton( min_seq_len = tl.min(seq_lens) if max_seq_len * 8 < min_seq_len * 10: min_seq_len = max_seq_len + if uneven_batch_ptr is not None: + tl.store(uneven_batch_ptr, (min_seq_len < max_seq_len).to(tl.int32)) max_kv_splits_1 = tl.minimum(tl.cdiv(max_seq_len, min_seq_len), max_kv_splits) kv_chunk_size_1 = tl.cdiv(max_seq_len, max_kv_splits_1) diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index 3d4ae2676e..ef19c8cece 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -143,6 +143,8 @@ class ForwardMetadata: lean_Lp: Optional[torch.Tensor] = None lean_Op: Optional[torch.Tensor] = None lean_locks: Optional[torch.Tensor] = None + # One int32, set by get_num_kv_splits when the decode seq lens are uneven + uneven_batch: Optional[torch.Tensor] = None class TritonAttnBackend(AttentionBackend): @@ -430,6 +432,7 @@ class TritonAttnBackend(AttentionBackend): self, num_kv_splits: torch.Tensor, seq_lens: torch.Tensor, + uneven_batch: Optional[torch.Tensor] = None, ): num_token, num_seq = num_kv_splits.shape[0], seq_lens.shape[0] # NOTE(alcanderian): Considering speculative_decodeing, @@ -473,6 +476,7 @@ class TritonAttnBackend(AttentionBackend): self.max_kv_splits, self.device_core_count, MAX_NUM_SEQ=SCHEDULE_SEQ, + uneven_batch_ptr=uneven_batch, ) def _dcp_lens(self, lens: torch.Tensor, start: Optional[torch.Tensor] = None): @@ -881,6 +885,7 @@ class TritonAttnBackend(AttentionBackend): # Lean decode buffers are only allocated on the decode path below; default # to None so the shared ForwardMetadata constructor works for extend/verify. lean_Mp = lean_Lp = lean_Op = lean_locks = None + uneven_batch = None if forward_batch.forward_mode.is_decode_or_idle(): if spec_info is None or spec_info.kv_indptr is None: @@ -964,6 +969,7 @@ class TritonAttnBackend(AttentionBackend): device=self.device, ) num_kv_splits = torch.empty((bs,), dtype=torch.int32, device=self.device) + uneven_batch = torch.zeros((1,), dtype=torch.int32, device=self.device) self.get_num_kv_splits( num_kv_splits, ( @@ -971,6 +977,7 @@ class TritonAttnBackend(AttentionBackend): if self.dcp_size > 1 else forward_batch.seq_lens ), + uneven_batch, ) # Lean decode persistent-grid partial-result buffers. @@ -1147,6 +1154,7 @@ class TritonAttnBackend(AttentionBackend): lean_Lp=lean_Lp, lean_Op=lean_Op, lean_locks=lean_locks, + uneven_batch=uneven_batch, ) def init_cuda_graph_state( @@ -1211,6 +1219,9 @@ class TritonAttnBackend(AttentionBackend): ) else: self.cuda_graph_num_kv_splits = cuda_graph_num_kv_splits_buf + self.cuda_graph_uneven_batch = torch.zeros( + (1,), dtype=torch.int32, device=self.device + ) if kv_indices_buf is None: self.cuda_graph_kv_indices = torch.zeros( @@ -1305,6 +1316,7 @@ class TritonAttnBackend(AttentionBackend): lean_Lp=self.cuda_graph_lean_Lp, lean_Op=self.cuda_graph_lean_Op, lean_locks=self.cuda_graph_lean_locks, + uneven_batch=self.cuda_graph_uneven_batch, ) elif forward_mode.is_target_verify(): custom_mask = ( @@ -1401,7 +1413,9 @@ class TritonAttnBackend(AttentionBackend): plan, bs, seq_lens, req_pool_indices ) self.get_num_kv_splits( - self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs] + self.cuda_graph_num_kv_splits[:bs], + num_kv_splits_lens[:bs], + self.cuda_graph_uneven_batch, ) if window_kv_lens is not None: self.get_num_kv_splits( @@ -2444,6 +2458,7 @@ class TritonAttnBackend(AttentionBackend): lean_Lp=self.forward_metadata.lean_Lp, lean_Op=self.forward_metadata.lean_Op, lean_locks=self.forward_metadata.lean_locks, + uneven_batch=self.forward_metadata.uneven_batch, ) local_lse = torch.logsumexp( self.forward_metadata.attn_lse[ @@ -2481,6 +2496,7 @@ class TritonAttnBackend(AttentionBackend): lean_Lp=self.forward_metadata.lean_Lp, lean_Op=self.forward_metadata.lean_Op, lean_locks=self.forward_metadata.lean_locks, + uneven_batch=self.forward_metadata.uneven_batch, ) return o diff --git a/test/registered/attention/test_triton_attention_kernels.py b/test/registered/attention/test_triton_attention_kernels.py index 06cc49d61a..55cd910560 100644 --- a/test/registered/attention/test_triton_attention_kernels.py +++ b/test/registered/attention/test_triton_attention_kernels.py @@ -19,6 +19,8 @@ from sglang.kernels.ops.attention.extend_attention import ( from sglang.kernels.ops.attention.prefill_attention import ( context_attention_fwd, ) +from sglang.srt.layers.attention.triton_backend import TritonAttnBackend +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.utils import get_device from sglang.srt.utils.common import temp_set_env from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -864,6 +866,108 @@ class TestTritonAttention(CustomTestCase): for B, H_Q, H_KV, D, D_V in configs: self._test_grouped_decode_attention_once(B, S, H_Q, H_KV, D, D_V) + def test_decode_attention_uneven_batch_graph_replay(self): + """One captured decode graph replayed over uniform, padded and ragged batches. + + TritonAttnBackend refreshes num_kv_splits and the uneven_batch flag in place + before each replay, and the captured stage-1 kernels read the flag at run time, + so every replay must see its own flag and match an eager run without rotation. + """ + device = get_device() + dtype = torch.bfloat16 + bs, pool, D, max_kv_splits = 8, 8192, 128, 8 + rounds = [ + ([600] * bs, 0), + ([600, 610, 620, 630, 640, 1, 1, 1], 1), # CUDA-graph padding rows + ([600] * bs, 0), + ([1000] + [64] * (bs - 1), 1), + ([700] * bs, 0), + ] + for H_Q, H_KV in ((8, 8), (16, 2)): + backend = TritonAttnBackend.__new__(TritonAttnBackend) + backend.num_head, backend.num_kv_head = H_Q, H_KV + backend.max_kv_splits = max_kv_splits + backend.device_core_count = torch.cuda.get_device_properties( + device + ).multi_processor_count + backend.static_kv_splits = False + backend.enable_deterministic = False + backend.split_tile_size = None + backend.cuda_graph_num_kv_splits = torch.zeros( + bs, dtype=torch.int32, device=device + ) + backend.cuda_graph_uneven_batch = torch.zeros( + 1, dtype=torch.int32, device=device + ) + backend._update_decode_kv_buffers = lambda plan, n, lens, req: ( + None, + None, + None, + lens, + ) + + seq_lens = torch.zeros(bs, dtype=torch.int32, device=device) + kv_indptr = torch.zeros(bs + 1, dtype=torch.int32, device=device) + kv_indices = torch.zeros(pool, dtype=torch.int64, device=device) + q = torch.randn(bs, H_Q, D, dtype=dtype, device=device) + k_buffer = torch.randn(pool, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(pool, H_KV, D, dtype=dtype, device=device) + o = torch.empty(bs, H_Q, D, dtype=dtype, device=device) + attn_logits = torch.empty( + bs, H_Q, max_kv_splits, D, dtype=torch.float32, device=device + ) + attn_lse = torch.empty( + bs, H_Q, max_kv_splits, dtype=torch.float32, device=device + ) + + def update(lens): + seq_lens.copy_(torch.tensor(lens, dtype=torch.int32)) + kv_indptr[1:] = torch.cumsum(seq_lens, 0) + kv_indices[: sum(lens)] = torch.randperm(pool, device=device)[ + : sum(lens) + ] + backend._apply_cuda_graph_metadata( + None, bs, None, seq_lens, ForwardMode.DECODE, None + ) + + def decode(out, uneven_batch): + decode_attention_fwd( + q, + k_buffer, + v_buffer, + out, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + backend.cuda_graph_num_kv_splits, + max_kv_splits, + D**-0.5, + 1.0, + 1.0, + enable_lean=False, + uneven_batch=uneven_batch, + ) + + update(rounds[1][0]) + decode(o, backend.cuda_graph_uneven_batch) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + decode(o, backend.cuda_graph_uneven_batch) + flag_ptr = backend.cuda_graph_uneven_batch.data_ptr() + for lens, uneven in rounds: + with self.subTest(H_Q=H_Q, H_KV=H_KV, lens=lens): + update(lens) + graph.replay() + ref = torch.empty_like(o) + decode(ref, None) + self.assertEqual( + backend.cuda_graph_uneven_batch.data_ptr(), flag_ptr + ) + self.assertEqual(backend.cuda_graph_uneven_batch.item(), uneven) + self.assertTrue(torch.equal(o, ref)) + def test_decode_attention_large_batch_int64_offset(self): """Regression for int32 Mid_O offset overflow (PR #28788).