From d5e83ea4f6a07c43d16774b1e329196cebde3280 Mon Sep 17 00:00:00 2001 From: Aarne Talman Date: Wed, 21 Oct 2026 02:11:01 +0300 Subject: [PATCH] [Perf] [AMD] deterministic: Faster extend attention launch on gfx950 (#45098) --- .../kernels/ops/attention/extend_attention.py | 6 +++ .../test_triton_attention_kernels.py | 51 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/python/sglang/kernels/ops/attention/extend_attention.py b/python/sglang/kernels/ops/attention/extend_attention.py index 025ec0d596..0d2aa44452 100644 --- a/python/sglang/kernels/ops/attention/extend_attention.py +++ b/python/sglang/kernels/ops/attention/extend_attention.py @@ -1561,6 +1561,12 @@ def extend_attention_fwd_unified( BLOCK_DMODEL, BLOCK_DPE, BLOCK_DV, BLOCK_M, BLOCK_N, num_warps = ( _get_block_sizes_for_extend_attention(Lq, Lv) ) + if _is_gfx95 and Lq <= 128 and Lv <= 128 and BLOCK_N == 64: + # gfx950, head_dim <= 128: each workgroup streams the whole prefix, so a 256-row query tile + # cuts the KV bytes read per query row 4x. BLOCK_N, and so the reduction order, is unchanged: + # outputs are bitwise identical. Measured on MI350X (head_dim 128 and 64, 8k-token chunks after + # 0-56k-token prefixes): 2.0-2.6x kernel speed. + BLOCK_M, num_warps = 256, 8 sm_scale = sm_scale or 1.0 / (Lq**0.5) batch_size, head_num = qo_indptr.shape[0] - 1, q.shape[1] diff --git a/test/registered/attention/test_triton_attention_kernels.py b/test/registered/attention/test_triton_attention_kernels.py index 55cd910560..163d7bc0d9 100644 --- a/test/registered/attention/test_triton_attention_kernels.py +++ b/test/registered/attention/test_triton_attention_kernels.py @@ -1,3 +1,4 @@ +import itertools import random import unittest @@ -1176,6 +1177,56 @@ class TestTritonAttention(CustomTestCase): B, N_CTX, H_Q, H_KV, D ) + def test_extend_attention_unified_gfx950_tile_bitwise(self): + """Deterministic mode relies on the gfx950 unified-kernel tile keeping the reduction + order: its output must equal the default tile's output bit for bit.""" + from sglang.kernels.ops.attention import extend_attention as ea + + if not ea._is_gfx95: + self.skipTest("gfx950-only tile") + device = get_device() + torch.manual_seed(0) + prefix_lens, extend_lens = [0, 1500, 4096], [700, 2048, 333] + for H_Q, H_KV, D in ((32, 8, 128), (64, 8, 64)): + seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)] + q = torch.randn( + sum(extend_lens), H_Q, D, dtype=torch.bfloat16, device=device + ) + k = torch.randn(sum(seq_lens), H_KV, D, dtype=torch.bfloat16, device=device) + v = torch.randn_like(k) + qo_indptr = torch.tensor( + [0, *itertools.accumulate(extend_lens)], + dtype=torch.int32, + device=device, + ) + kv_indptr = torch.tensor( + [0, *itertools.accumulate(seq_lens)], dtype=torch.int32, device=device + ) + kv_indices = torch.arange(sum(seq_lens), dtype=torch.int64, device=device) + prefix = torch.tensor(prefix_lens, dtype=torch.int32, device=device) + + def run(): + o = torch.empty_like(q) + extend_attention_fwd_unified( + q, + o, + k, + v, + 1.0, + 1.0, + qo_indptr, + kv_indptr, + kv_indices, + prefix, + max_len_extend=max(extend_lens), + ) + return o + + o_tuned = run() + with unittest.mock.patch.object(ea, "_is_gfx95", False): + o_default = run() + self.assertTrue(torch.equal(o_tuned, o_default), f"head_dim {D}") + def test_build_unified_kv_indices(self): """Test build_unified_kv_indices correctness.""" B = 4