From 37a47737c805782028e2c7cdbcc99f8770c0774c Mon Sep 17 00:00:00 2001 From: anthonyhe-bot Date: Tue, 29 Sep 2026 18:03:33 -0400 Subject: [PATCH] [Perf] Tune SM90 GDN recurrent verify launch for small batches (#41486) Co-authored-by: Claude Opus 5.5 Co-authored-by: hnyls2002 Co-authored-by: Liangsheng Yin --- .../fla/fused_sigmoid_gating_recurrent.py | 21 +++++- .../attention/test_fused_verify_triton_gdn.py | 73 ++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py index 3d6758db13..1fdfe7df1c 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py +++ b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py @@ -5,10 +5,11 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.srt.utils import is_gfx95_supported, is_hip, is_sm90_supported _is_hip = is_hip() _is_gfx95 = is_gfx95_supported() +_is_sm90 = is_sm90_supported() def _select_recurrent_launch_config( @@ -18,6 +19,7 @@ def _select_recurrent_launch_config( k: int, v: int, is_kda: bool, + target_verify: bool = False, ) -> tuple[int, int]: """Select the value tile and warp count for recurrent GDN.""" if ( @@ -31,6 +33,19 @@ def _select_recurrent_launch_config( and v == 128 ): return (8, 4) if n == 1 else (16, 2) + if ( + target_verify + and not _is_hip + and not is_kda + and 0 < n <= 64 + and k == 128 + and v == 128 + and _is_sm90 + ): + # BV=4 and n <= 64 measured on H100/H200. SM90 only: Blackwell is faster + # with narrow tiles but not bit-identical to BV=32. Only the dense + # intermediate-state verify sets target_verify; cache_ring keeps BV=32. + return 4, 1 return min(triton.next_power_of_2(v), 32), 1 @@ -428,7 +443,9 @@ def fused_sigmoid_gating_delta_rule_update( stride_a = a.stride()[1] if a.ndim == 4 else a.stride()[-2] HV = v.shape[2] N = B if cu_seqlens is None else len(cu_seqlens) - 1 - BV, num_warps = _select_recurrent_launch_config(N, H, HV, K, V, is_kda) + BV, num_warps = _select_recurrent_launch_config( + N, H, HV, K, V, is_kda, target_verify=intermediate_states_buffer is not None + ) BK = triton.next_power_of_2(K) NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) assert NK == 1, "NK > 1 is not supported yet" diff --git a/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py b/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py index f66a893203..55f434fdf5 100644 --- a/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py +++ b/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py @@ -11,10 +11,13 @@ import sys import pytest import torch -from sglang.srt.utils import is_gfx95_supported +from sglang.srt.utils import is_gfx95_supported, is_sm90_supported from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci try: + from sglang.kernels.ops.attention.fla import ( + fused_sigmoid_gating_recurrent as recurrent_module, + ) from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_gated_delta_rule_update, @@ -233,6 +236,74 @@ def test_qwen35_tp4_launch_config_is_narrow(): assert _select_recurrent_launch_config(3, 4, 16, 128, 128, True) == (32, 1) +_requires_sm90 = pytest.mark.skipif( + torch.version.hip is not None or not is_sm90_supported(), reason="requires SM90" +) + + +@_requires_sm90 +def test_sm90_verify_launch_config_is_narrow(): + assert _select_recurrent_launch_config(1, 16, 32, 128, 128, False, True) == (4, 1) + assert _select_recurrent_launch_config(64, 4, 8, 128, 128, False, True) == (4, 1) + assert _select_recurrent_launch_config(65, 16, 32, 128, 128, False, True) == (32, 1) + assert _select_recurrent_launch_config(1, 16, 32, 128, 128, False, False) == (32, 1) + assert _select_recurrent_launch_config(1, 16, 32, 128, 128, True, True) == (32, 1) + assert _select_recurrent_launch_config(1, 16, 32, 64, 64, False, True) == (32, 1) + + +@_requires_sm90 +@pytest.mark.skipif(not KERNELS_AVAILABLE, reason="Kernels not available") +@pytest.mark.parametrize("N", [1, 3, 16, 64]) +@pytest.mark.parametrize("T", [4, 16]) +@pytest.mark.parametrize("H,HV", [(16, 32), (4, 16)]) +@pytest.mark.parametrize("tree", [False, True]) +def test_sm90_verify_launch_is_bit_exact( + N: int, T: int, H: int, HV: int, tree: bool, monkeypatch +): + """The tuned SM90 verify launch matches the BV=32 launch bit for bit.""" + K = V = 128 + A_log, dt_bias, a, b, q, k, v, state, indices, cu_seqlens = _make_tensors( + N, T, H, HV, K, V + ) + retrieve_parent_token = None + if tree: + # Random draft tree: token i > 0 reads the cached state of an earlier token. + steps = torch.arange(T, device="cuda") + retrieve_parent_token = (torch.rand(N, T, device="cuda") * steps).long() + retrieve_parent_token[:, 0] = -1 + + def run(launch_config): + monkeypatch.setattr( + recurrent_module, "_select_recurrent_launch_config", launch_config + ) + buffer = torch.zeros(N, T, HV, V, K, dtype=torch.float32, device="cuda") + out = run_fused_mtp( + A_log, + dt_bias, + q, + k, + v, + a, + b, + state.clone(), + indices, + cu_seqlens, + disable_state_update=True, + intermediate_states_buffer=buffer, + intermediate_state_indices=indices, + cache_steps=T, + retrieve_parent_token=retrieve_parent_token, + ) + return out, buffer + + out_tuned, states_tuned = run(_select_recurrent_launch_config) + out_default, states_default = run(lambda *args, **kwargs: (32, 1)) + + assert _select_recurrent_launch_config(N, H, HV, K, V, False, True) == (4, 1) + assert torch.equal(out_tuned, out_default) + assert torch.equal(states_tuned, states_default) + + @pytest.mark.skipif(not KERNELS_AVAILABLE, reason="Kernels not available") @pytest.mark.parametrize("N", [1, 16, 128]) def test_mtp_single_step_decode(N: int):