From 71dc292b9702ed08c8087924edad2fbe248cc061 Mon Sep 17 00:00:00 2001 From: zijie Date: Sat, 3 Oct 2026 17:54:49 -0700 Subject: [PATCH] [AMD] Use AITER ASM prefill for MiniMax-M3 HD128 attention (#41707) Co-authored-by: Zijie Chen <300606707+zijiecode@users.noreply.github.com> Co-authored-by: Thomas Wang --- python/sglang/srt/environ.py | 2 + .../srt/layers/attention/aiter_backend.py | 52 ++++++ .../amd/test_minimax_hd128_asm_prefill.py | 157 ++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 test/registered/amd/test_minimax_hd128_asm_prefill.py diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 031a429099..810473859b 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -579,6 +579,8 @@ class Envs: # of the occupancy-starved mha_batch_prefill FMHA. Independent kill-switch # for the new path; pairs with SGLANG_AITER_UNIFIED_VERIFY. Default on. SGLANG_AITER_UNIFIED_DRAFT_EXTEND = EnvBool(True) + # Use ASM prefill for gfx950 HD128 FP8 KV, including cached prefixes. + SGLANG_AITER_ASM_PREFILL_HD128 = EnvBool(True) # Attention (aiter, ROCm): hand chunked prefill the page-level KV view so # gfx950 fp8 hd256 takes aiter's paged-varlen asm kernel. That kernel is # compiled for 4D LINEAR [N, 64, H, D], so it serves --page-size 64 and diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 81a67efe73..287185f9aa 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -3735,6 +3735,58 @@ class AiterAttnBackend(AttentionBackend): ) ) + if ( + envs.SGLANG_AITER_ASM_PREFILL_HD128.get() + and is_gfx95_supported() + and forward_batch.forward_mode.is_extend() + and not layer.is_cross_attention + and window_size == (-1, -1) + and sinks is None + and self.logits_soft_cap == 0.0 + and layer.qk_head_dim == layer.v_head_dim == 128 + and layer.tp_k_head_num == layer.tp_v_head_num + and self.kv_cache_dtype == fp8_dtype + and q.dtype == torch.bfloat16 + and _aiter_fp8_asm_supports_gqa( + layer.tp_q_head_num, layer.tp_k_head_num + ) + and not self.kv_cache_is_vectorized_5d + and self.forward_metadata.max_kv_len is not None + ): + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) + tok_idx, cu_k = self._asm_context_prefill_indices( + forward_batch, forward_batch.batch_size, k_cache.shape[0] + ) + if tok_idx is not None: + # Read the already quantized cache for first and later + # chunks alike. Raw K/V may come from different producers; + # casting them here would skip or repeat their KV scaling. + k_gather = ( + k_cache.view(torch.uint8) + .index_select(0, tok_idx) + .view(fp8_dtype) + ) + v_gather = ( + v_cache.view(torch.uint8) + .index_select(0, tok_idx) + .view(fp8_dtype) + ) + o = flash_attn_varlen_fp8_pertensor_func( + q.contiguous().view(-1, layer.tp_q_head_num, 128).to(fp8_dtype), + k_gather, + v_gather, + self.k_scale.reshape(1), # Q is cast at unit scale. + k_descale.reshape(1), + v_descale.reshape(1), + self.qo_indptr[:bs0], + cu_k, + self.forward_metadata.max_q_len, + int(self.forward_metadata.max_kv_len), + softmax_scale=layer.scaling, + causal=True, + ) + return o.to(self.input_dtype).view(-1, layer.tp_q_head_num * 128) + # Context-chunk prefill (extend batches WITH a prefix) via the # gfx950 ASM fp8 varlen fmha. The ck_tile paged batch_prefill runs # at ~15% FP8 MFU at these shapes while the ASM kernel is ~3.5x diff --git a/test/registered/amd/test_minimax_hd128_asm_prefill.py b/test/registered/amd/test_minimax_hd128_asm_prefill.py new file mode 100644 index 0000000000..daafea959c --- /dev/null +++ b/test/registered/amd/test_minimax_hd128_asm_prefill.py @@ -0,0 +1,157 @@ +"""Exercise the SGLang HD128 route with quantized KV and non-unit scales.""" + +import os +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.test.ci.ci_register import register_amd_ci + +register_amd_ci(est_time=45, suite="stage-b-test-1-gpu-small-amd") + + +@unittest.skipUnless(torch.version.hip, "ROCm ASM prefill") +class TestMiniMaxHD128AsmPrefill(unittest.TestCase): + @classmethod + def setUpClass(cls): + from sglang.srt.layers.attention import aiter_backend + from sglang.srt.model_executor.forward_batch_info import ForwardMode + + if "gfx950" not in torch.cuda.get_device_properties(0).gcnArchName: + raise unittest.SkipTest("HD128 ASM prefill requires gfx950") + cls.ab = aiter_backend + cls.extend_mode = ForwardMode.EXTEND + + def exercise(self, q_lens, k_lens, kv_heads=1): + torch.manual_seed(4102) + torch.backends.cuda.matmul.allow_tf32 = False + device = "cuda" + dtype = self.ab.fp8_dtype + q_heads = kv_heads * 16 + slots = sum(k_lens) + 128 + token_ids = torch.randperm(slots, device=device)[: sum(k_lens)].to(torch.int32) + k_cache = torch.randn(slots, kv_heads, 128, device=device).to(dtype) + v_cache = torch.randn(slots, kv_heads, 128, device=device).to(dtype) + q = torch.randn(sum(q_lens), q_heads, 128, device=device, dtype=torch.bfloat16) + k_scale = torch.tensor([0.25], dtype=torch.float32, device=device) + v_scale = torch.tensor([1.5], dtype=torch.float32, device=device) + cu_q = torch.tensor( + [0, *torch.tensor(q_lens).cumsum(0).tolist()], + device=device, + dtype=torch.int32, + ) + cu_k = torch.tensor( + [0, *torch.tensor(k_lens).cumsum(0).tolist()], + device=device, + dtype=torch.int32, + ) + backend = self.ab.AiterAttnBackend.__new__(self.ab.AiterAttnBackend) + backend.use_mla = False + backend._use_unified_verify = False + backend.kv_cache_dtype = dtype + backend.kv_cache_is_vectorized_5d = False + backend.input_dtype = torch.bfloat16 + backend.k_scale = torch.ones(1, device=device) + backend.v_scale = torch.ones(1, device=device) + backend.qo_indptr = cu_q + backend.token_to_kv_pool = SimpleNamespace( + get_kv_buffer=lambda _: (k_cache, v_cache) + ) + backend.forward_metadata = self.ab.ForwardMetadata( + kv_indptr=cu_k, + kv_indices=token_ids, + qo_indptr=cu_q, + kv_last_page_len=None, + max_q_len=max(q_lens), + max_kv_len=max(k_lens), + ) + batch = SimpleNamespace( + batch_size=len(q_lens), + forward_mode=self.extend_mode, + attn_attend_prefix_cache=False, + out_cache_loc=None, + spec_info=None, + seq_lens_cpu=torch.tensor(k_lens, dtype=torch.int32), + seq_lens=torch.tensor(k_lens, dtype=torch.int32, device=device), + extend_prefix_lens_cpu=[k - q for k, q in zip(k_lens, q_lens)], + ) + layer = SimpleNamespace( + layer_id=0, + logit_cap=0.0, + is_cross_attention=False, + sliding_window_size=-1, + qk_head_dim=128, + v_head_dim=128, + tp_q_head_num=q_heads, + tp_k_head_num=kv_heads, + tp_v_head_num=kv_heads, + scaling=128**-0.5, + k_scale=k_scale, + v_scale=v_scale, + ) + # Deliberately different raw K/V: the route must read the quantized pool, + # including on the first chunk, rather than cast these inputs again. + raw_k = torch.zeros( + sum(q_lens), kv_heads, 128, device=device, dtype=torch.bfloat16 + ) + raw_v = torch.zeros_like(raw_k) + with patch.dict(os.environ): + os.environ.pop("SGLANG_AITER_ASM_PREFILL_HD128", None) + with patch.object( + self.ab, + "flash_attn_varlen_fp8_pertensor_func", + wraps=self.ab.flash_attn_varlen_fp8_pertensor_func, + ) as asm: + output = backend.forward_extend( + q, raw_k, raw_v, layer, batch, save_kv_cache=False + ).view_as(q) + self.assertEqual(asm.call_count, 1) + + query_offset = key_offset = 0 + for q_len, k_len in zip(q_lens, k_lens): + sample = sorted({0, q_len // 2, q_len - 1}) + physical = token_ids[key_offset : key_offset + k_len].long() + keys = k_cache.view(torch.uint8)[physical].view(dtype).float() * k_scale + values = v_cache.view(torch.uint8)[physical].view(dtype).float() * v_scale + limit = k_len - q_len + torch.tensor(sample, device=device) + 1 + allowed = torch.arange(k_len, device=device)[None, :] < limit[:, None] + for head in range(kv_heads): + head_slice = slice(head * 16, (head + 1) * 16) + query = ( + q[query_offset : query_offset + q_len][sample, head_slice] + .to(dtype) + .float() + ) + scores = ( + torch.einsum("qhd,kd->hqk", query, keys[:, head]) * layer.scaling + ) + scores.masked_fill_(~allowed[None, :, :], -float("inf")) + reference = torch.einsum( + "hqk,kd->qhd", scores.softmax(-1), values[:, head] + ) + actual = output[query_offset : query_offset + q_len][ + sample, head_slice + ].float() + self.assertLess( + (actual - reference).abs().max().item(), 0.055 * v_scale.item() + ) + self.assertLess( + ((actual - reference).norm() / reference.norm()).item(), 0.05 + ) + query_offset += q_len + key_offset += k_len + + def test_first_chunk_reads_fp8_pool(self): + self.exercise([7, 3], [7, 3]) + + def test_ragged_context_and_two_kv_heads(self): + self.exercise([4, 2], [129, 257], kv_heads=2) + + def test_one_million_token_context(self): + self.exercise([4], [1048576]) + + +if __name__ == "__main__": + unittest.main()