From 0e2aac500bbd7010503bff67bd221bfbd7f04389 Mon Sep 17 00:00:00 2001 From: Olga Miroshnichenko Date: Sat, 26 Sep 2026 05:23:46 +0300 Subject: [PATCH] [ROCm][Perf] aiter: page-level KV view for gfx950 fp8 page-64 asm prefill (#36505) --- .../docs/references/environment_variables.mdx | 5 + python/sglang/srt/environ.py | 10 ++ .../srt/layers/attention/aiter_backend.py | 103 +++++++++++++++++- test/manual/test_aiter_paged_kv_view.py | 97 +++++++++++++++++ 4 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 test/manual/test_aiter_paged_kv_view.py diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index f013ac4ce4..b930c847f0 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -1857,6 +1857,11 @@ SGLang supports various environment variables that can be used to configure its Physical layout for the MHA KV cache on AITER: nhd or vectorized_5d (SHUFFLE layout enabling pa_decode_gluon). nhd + + SGLANG_AITER_PAGED_PREFILL_ASM + Let gfx950 fp8 hd256 chunked prefill read the KV pool in place through AITER's paged-varlen ASM kernel, instead of gathering the prefix into a contiguous buffer first. The ASM kernel is compiled for a 4D LINEAR [N, 64, H, D] KV cache, so it serves --page-size 64 on the NHD pool and nothing else; at any other page size this variable has no effect whichever way it is set. Where it does apply, the gain grows with context and is flat to marginally negative at moderate ISL, so set it to false for workloads that are not long-context. + true + SGLANG_ROCM_FUSED_DECODE_MLA Use the fused decode MLA kernel on ROCm. diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a5810505e6..d6c7e0da20 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -576,6 +576,16 @@ class Envs: # of the occupancy-starved mha_batch_prefill FMHA. Independent kill-switch # for the new path; pairs with SGLANG_AITER_UNIFIED_VERIFY. Default on. SGLANG_AITER_UNIFIED_DRAFT_EXTEND = EnvBool(True) + # Attention (aiter, ROCm): hand chunked prefill the page-level KV view so + # gfx950 fp8 hd256 takes aiter's paged-varlen asm kernel. That kernel is + # compiled for 4D LINEAR [N, 64, H, D], so it serves --page-size 64 and + # nothing else; at any other page size the view is never built and this + # flag does nothing whichever way it is set. Where it does apply the win + # grows with context (~1-2% throughput and 3-7% TTFT at 60k and above) and + # is flat to marginally negative at moderate ISL, so this is a per-workload + # switch rather than a pure kill-switch. Off falls back to the contiguous + # gather path. Default on. + SGLANG_AITER_PAGED_PREFILL_ASM = EnvBool(True) # size the KV pool after CUDA-graph capture SGLANG_ENABLE_POST_CAPTURE_KV_SIZING = EnvBool(False) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 4809bbcb13..a9ae9476b5 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -164,6 +164,44 @@ class ForwardMetadata: asm_ctx_ready: bool = False asm_ctx_tok_idx: Optional[torch.Tensor] = None asm_ctx_cu_k: Optional[torch.Tensor] = None + # (page_indptr, page_ids, last_page_len); None means use the token-level table + paged_kv_view: Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None + + +def _build_paged_kv_view( + kv_indices: torch.Tensor, seq_lens_cpu: torch.Tensor, page_size: int +): + """Token-level flashinfer KV table -> the page-level view aiter's asm + paged-varlen prefill expects. + + The paged allocator packs each page contiguously, so a token slot's page id + is slot // page_size. The arithmetic runs on seq_lens_cpu so it costs no + device sync; only the gather touches the GPU. + """ + device = kv_indices.device + seq_lens = seq_lens_cpu.to(torch.int64) + pages = (seq_lens + page_size - 1) // page_size + + page_indptr = torch.zeros(pages.numel() + 1, dtype=torch.int32) + page_indptr[1:] = torch.cumsum(pages, dim=0) + tok_base = torch.zeros(pages.numel() + 1, dtype=torch.int64) + tok_base[1:] = torch.cumsum(seq_lens, dim=0) + + within = torch.arange( + int(page_indptr[-1]), dtype=torch.int64 + ) - torch.repeat_interleave(page_indptr[:-1].to(torch.int64), pages) + gather = torch.repeat_interleave(tok_base[:-1], pages) + within * page_size + page_ids = (kv_indices[gather.to(device)] // page_size).to(torch.int32) + last_page_len = ((seq_lens - 1) % page_size + 1).to(torch.int32) + return page_indptr.to(device), page_ids, last_page_len.to(device) + + +def _paged_prefill_asm_supports_gqa(num_q_heads: int, num_kv_heads: int) -> bool: + """aiter's asm paged-varlen guard takes only a power-of-two GQA ratio.""" + if num_kv_heads <= 0 or num_q_heads % num_kv_heads != 0: + return False + gqa = num_q_heads // num_kv_heads + return gqa & (gqa - 1) == 0 _AITER_PARTITION_SIZE_ROCM = 256 @@ -2030,6 +2068,25 @@ class AiterAttnBackend(AttentionBackend): self.indices_updater_prefill.kv_indices ).to(torch.int32) + # Once per batch, not per layer: forward_extend only consumes it. + # The arch test is not redundant with the flag: the asm guard is + # gfx95-only, and elsewhere there is no kernel for this shape at + # all, so building a view we must not pass is wasted work. + paged_kv_view = None + if ( + envs.SGLANG_AITER_PAGED_PREFILL_ASM.get() + and is_gfx95_supported() + and self.page_size == 64 + and not self.kv_cache_is_vectorized_5d + and not self.use_sliding_window_kv_pool + and not self.use_triton_unified_attention + ): + paged_kv_view = _build_paged_kv_view( + self.indices_updater_prefill.kv_indices, + forward_batch.seq_lens_cpu, + self.page_size, + ) + self.forward_metadata = ForwardMetadata( self.indices_updater_prefill.kv_indptr, self.indices_updater_prefill.kv_indices, @@ -2039,6 +2096,7 @@ class AiterAttnBackend(AttentionBackend): forward_batch.seq_lens_cpu.max().item(), swa_page_table=swa_page_table, swa_out_cache_loc=swa_out_cache_loc, + paged_kv_view=paged_kv_view, ) def _plan_dcp_decode_metadata( @@ -3543,6 +3601,18 @@ class AiterAttnBackend(AttentionBackend): if layer.sliding_window_size is not None and layer.sliding_window_size > -1: window_size = (layer.sliding_window_size, -1) + # Whether the paged asm prefill kernel can serve this layer on this + # batch. Both the gather branch below and the paged branch at the + # end of this method key off it, so the two stay mutually exclusive + # by construction: one flag, read twice, cannot drift out of sync + # the way two copies of the same condition would. + paged_asm_available = ( + self.forward_metadata.paged_kv_view is not None + and _paged_prefill_asm_supports_gqa( + layer.tp_q_head_num, layer.tp_k_head_num + ) + ) + # Context-chunk prefill (extend batches WITH a prefix) via the # gfx950 ASM fp8 varlen fmha. The ck_tile paged batch_prefill runs # at ~15% FP8 MFU at these shapes while the ASM kernel is ~3.5x @@ -3551,6 +3621,12 @@ class AiterAttnBackend(AttentionBackend): # first chunk already takes the ASM branch below. # This applies to Qwen3.5 full-attention layers only currently, # Other configurations fall through to the attention paths below. + # + # Yields to the paged asm kernel when there is one, since that reads + # the cache in place and does the same attention with no gather at + # all. This is not a blanket disable: on any batch or layer that + # kernel cannot serve, the flag is False and this branch runs + # exactly as it does today. if ( is_gfx95_supported() and forward_batch.forward_mode.is_extend() @@ -3567,6 +3643,7 @@ class AiterAttnBackend(AttentionBackend): ) and not self.kv_cache_is_vectorized_5d and self.forward_metadata.max_kv_len is not None + and not paged_asm_available ): bs = forward_batch.batch_size k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) @@ -3704,12 +3781,36 @@ class AiterAttnBackend(AttentionBackend): -1, layer.tp_q_head_num, layer.head_dim ) + kv_indptr_arg = self.forward_metadata.kv_indptr[:bs0] + if ( + paged_asm_available + and page_table is self.forward_metadata.kv_indices + and window_size == (-1, -1) + and sinks is None + and self.logits_soft_cap == 0.0 + and layer.qk_head_dim == 256 + and layer.v_head_dim == 256 + and self.kv_cache_dtype == fp8_dtype + ): + # These must match aiter's asm guard: there is no CK arm for 4D + # LINEAR page-64 fp8 hd256, so a shape the guard rejects raises + # rather than falling back. The arch half of the guard is + # checked where paged_kv_view is built. + page_indptr, page_ids, last_page_len = ( + self.forward_metadata.paged_kv_view + ) + kv_indptr_arg = page_indptr[:bs0] + page_table = page_ids + k_cache = k_cache.view(-1, self.page_size, *k_cache.shape[-2:]) + v_cache = v_cache.view(-1, self.page_size, *v_cache.shape[-2:]) + extra_kwargs["kv_last_page_lens"] = last_page_len[: bs0 - 1] + o = mha_batch_prefill_func( q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), k_cache, v_cache, self.qo_indptr[:bs0], - self.forward_metadata.kv_indptr[:bs0], + kv_indptr_arg, page_table, self.forward_metadata.max_q_len, self.forward_metadata.max_kv_len, diff --git a/test/manual/test_aiter_paged_kv_view.py b/test/manual/test_aiter_paged_kv_view.py new file mode 100644 index 0000000000..5a28966970 --- /dev/null +++ b/test/manual/test_aiter_paged_kv_view.py @@ -0,0 +1,97 @@ +""" +Unit test for the aiter paged-prefill KV view conversion. + +aiter_backend._build_paged_kv_view turns the token-level flashinfer KV table +into the page-level view aiter's fp8 hd256 page-64 paged-varlen asm prefill +expects. This checks the index arithmetic against a shuffled page allocation, so +a wrong page id or last-page length is caught. The tensors stay on CPU (no GPU +needed, though the import pulls in the usual sglang deps); the kernel path needs +ROCm/gfx950 and is covered by the evidence in the PR. + +Run: + python test/manual/test_aiter_paged_kv_view.py +""" + +import os +import sys +import unittest + +import torch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../python")) + + +class TestAiterPagedKVView(unittest.TestCase): + def setUp(self): + from sglang.srt.layers.attention.aiter_backend import _build_paged_kv_view + + self.build = _build_paged_kv_view + self.page_size = 64 + + def _table(self, page_ids_per_req, seq_lens): + """The token-level table create_flashinfer_kv_indices_triton produces: + per request, the KV slots of its pages in page order.""" + offs = torch.arange(self.page_size, dtype=torch.int64) + rows = [] + for pages, seq_len in zip(page_ids_per_req, seq_lens): + slots = (torch.tensor(pages, dtype=torch.int64) * self.page_size).unsqueeze( + -1 + ) + offs + rows.append(slots.reshape(-1)[:seq_len]) + return torch.cat(rows).to(torch.int32) + + def test_shuffled_pages_and_ragged_lengths(self): + # A real allocator hands out pages out of order, so page id != position + # in the table; an identity table would pass even if this were wrong. + page_ids_per_req = [[7, 3, 11], [5, 2]] + seq_lens = [130, 128] # 130 -> 3 pages, last one holding 2 tokens + kv_indices = self._table(page_ids_per_req, seq_lens) + + page_indptr, page_ids, last_page_len = self.build( + kv_indices, torch.tensor(seq_lens), self.page_size + ) + + self.assertEqual(page_indptr.tolist(), [0, 3, 5]) + self.assertEqual(page_ids.tolist(), [7, 3, 11, 5, 2]) + self.assertEqual(last_page_len.tolist(), [2, 64]) + self.assertEqual(page_indptr.dtype, torch.int32) + self.assertEqual(page_ids.dtype, torch.int32) + self.assertEqual(last_page_len.dtype, torch.int32) + + def test_exact_multiple_uses_full_last_page(self): + kv_indices = self._table([[4, 9]], [128]) + _, page_ids, last_page_len = self.build( + kv_indices, torch.tensor([128]), self.page_size + ) + self.assertEqual(page_ids.tolist(), [4, 9]) + self.assertEqual(last_page_len.tolist(), [64]) + + +class TestPagedPrefillAsmSupportsGqa(unittest.TestCase): + """The asm guard rejects a non-power-of-two GQA ratio, and a rejected call + raises rather than falling back, so the backend has to agree with it.""" + + def setUp(self): + from sglang.srt.layers.attention.aiter_backend import ( + _paged_prefill_asm_supports_gqa, + ) + + self.supported = _paged_prefill_asm_supports_gqa + + def test_power_of_two_ratios_are_supported(self): + for num_q_heads, num_kv_heads in [(1, 1), (4, 1), (8, 1), (16, 2), (32, 4)]: + with self.subTest(q=num_q_heads, kv=num_kv_heads): + self.assertTrue(self.supported(num_q_heads, num_kv_heads)) + + def test_non_power_of_two_ratios_are_rejected(self): + for num_q_heads, num_kv_heads in [(12, 2), (24, 4), (40, 8), (6, 1)]: + with self.subTest(q=num_q_heads, kv=num_kv_heads): + self.assertFalse(self.supported(num_q_heads, num_kv_heads)) + + def test_indivisible_and_degenerate_head_counts_are_rejected(self): + self.assertFalse(self.supported(8, 3)) + self.assertFalse(self.supported(8, 0)) # must not raise ZeroDivisionError + + +if __name__ == "__main__": + unittest.main()