[Perf] Tune SM90 GDN recurrent verify launch for small batches (#41486)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
anthonyhe-bot
2026-09-29 15:03:33 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 hnyls2002 Liangsheng Yin
parent 4885b563c3
commit 37a47737c8
2 changed files with 91 additions and 3 deletions
@@ -5,10 +5,11 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.utils import is_gfx95_supported, is_hip
from sglang.srt.utils import is_gfx95_supported, is_hip, is_sm90_supported
_is_hip = is_hip()
_is_gfx95 = is_gfx95_supported()
_is_sm90 = is_sm90_supported()
def _select_recurrent_launch_config(
@@ -18,6 +19,7 @@ def _select_recurrent_launch_config(
k: int,
v: int,
is_kda: bool,
target_verify: bool = False,
) -> tuple[int, int]:
"""Select the value tile and warp count for recurrent GDN."""
if (
@@ -31,6 +33,19 @@ def _select_recurrent_launch_config(
and v == 128
):
return (8, 4) if n == 1 else (16, 2)
if (
target_verify
and not _is_hip
and not is_kda
and 0 < n <= 64
and k == 128
and v == 128
and _is_sm90
):
# BV=4 and n <= 64 measured on H100/H200. SM90 only: Blackwell is faster
# with narrow tiles but not bit-identical to BV=32. Only the dense
# intermediate-state verify sets target_verify; cache_ring keeps BV=32.
return 4, 1
return min(triton.next_power_of_2(v), 32), 1
@@ -428,7 +443,9 @@ def fused_sigmoid_gating_delta_rule_update(
stride_a = a.stride()[1] if a.ndim == 4 else a.stride()[-2]
HV = v.shape[2]
N = B if cu_seqlens is None else len(cu_seqlens) - 1
BV, num_warps = _select_recurrent_launch_config(N, H, HV, K, V, is_kda)
BV, num_warps = _select_recurrent_launch_config(
N, H, HV, K, V, is_kda, target_verify=intermediate_states_buffer is not None
)
BK = triton.next_power_of_2(K)
NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
assert NK == 1, "NK > 1 is not supported yet"