[Perf] Tune SM90 GDN recurrent verify launch for small batches (#41486)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
hnyls2002
Liangsheng Yin
parent
4885b563c3
commit
37a47737c8
@@ -5,10 +5,11 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip, is_sm90_supported
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_gfx95 = is_gfx95_supported()
|
||||
_is_sm90 = is_sm90_supported()
|
||||
|
||||
|
||||
def _select_recurrent_launch_config(
|
||||
@@ -18,6 +19,7 @@ def _select_recurrent_launch_config(
|
||||
k: int,
|
||||
v: int,
|
||||
is_kda: bool,
|
||||
target_verify: bool = False,
|
||||
) -> tuple[int, int]:
|
||||
"""Select the value tile and warp count for recurrent GDN."""
|
||||
if (
|
||||
@@ -31,6 +33,19 @@ def _select_recurrent_launch_config(
|
||||
and v == 128
|
||||
):
|
||||
return (8, 4) if n == 1 else (16, 2)
|
||||
if (
|
||||
target_verify
|
||||
and not _is_hip
|
||||
and not is_kda
|
||||
and 0 < n <= 64
|
||||
and k == 128
|
||||
and v == 128
|
||||
and _is_sm90
|
||||
):
|
||||
# BV=4 and n <= 64 measured on H100/H200. SM90 only: Blackwell is faster
|
||||
# with narrow tiles but not bit-identical to BV=32. Only the dense
|
||||
# intermediate-state verify sets target_verify; cache_ring keeps BV=32.
|
||||
return 4, 1
|
||||
return min(triton.next_power_of_2(v), 32), 1
|
||||
|
||||
|
||||
@@ -428,7 +443,9 @@ def fused_sigmoid_gating_delta_rule_update(
|
||||
stride_a = a.stride()[1] if a.ndim == 4 else a.stride()[-2]
|
||||
HV = v.shape[2]
|
||||
N = B if cu_seqlens is None else len(cu_seqlens) - 1
|
||||
BV, num_warps = _select_recurrent_launch_config(N, H, HV, K, V, is_kda)
|
||||
BV, num_warps = _select_recurrent_launch_config(
|
||||
N, H, HV, K, V, is_kda, target_verify=intermediate_states_buffer is not None
|
||||
)
|
||||
BK = triton.next_power_of_2(K)
|
||||
NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
|
||||
assert NK == 1, "NK > 1 is not supported yet"
|
||||
|
||||
Reference in New Issue
Block a user